{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from sklearn.metrics import accuracy_score\n",
    "from xgboost import XGBClassifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一般模式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[17:10:22] 6513x127 matrix with 143286 entries loaded from ./data/agaricus.txt.train\n",
      "[17:10:22] 1611x127 matrix with 35442 entries loaded from ./data/agaricus.txt.test\n",
      "0.9929372025180409\n",
      "0.9937926753569212\n"
     ]
    }
   ],
   "source": [
    "path = './data/'\n",
    "#准备数据 数据下载在xgboost的github上 /demo/data/\n",
    "dtrain = xgb.DMatrix(path+'agaricus.txt.train')\n",
    "dtest = xgb.DMatrix(path+'agaricus.txt.test')\n",
    "#准备参数\n",
    "param = {'max_depth':2,'eta':1,'silent':0,'objective':'binary:logistic'}\n",
    "#参数说明\n",
    "'''\n",
    "max_depth:树的最大深度 缺省值为6 取值范围[1,+无穷]\n",
    "eta:防止过拟合在更新的时候使用的收缩步长 缺省值为0.3 取值[0,1] 其实就是学习率\n",
    "silent:0表示运行时打印运行信息，1表示不打印运行信息 缺省值为0\n",
    "objective:定义学习任务和学习目标 binary:logistic表示二分类逻辑回归 输出为概率\n",
    "'''\n",
    "#设置Boosting的迭代轮数 也就是最后会用到多少颗树\n",
    "num_round = 3\n",
    "#训练\n",
    "bst = xgb.train(param,dtrain,num_round)\n",
    "\n",
    "def evaluate(data,model):\n",
    "    pre = model.predict(data)\n",
    "    pre_res = [round(res) for res in pre]\n",
    "    y = data.get_label()\n",
    "    acc = accuracy_score(y,pre_res)\n",
    "    return acc\n",
    "\n",
    "acc_train = evaluate(dtrain,bst)\n",
    "acc_test = evaluate(dtest,bst)\n",
    "print(acc_train)\n",
    "print(acc_test)\n",
    "\n",
    "# 0.9777368340242591\n",
    "# 0.978274363749224\n",
    "\n",
    "# #训练结束之后进行预测（在训练样本上）\n",
    "# pre_train = bst.predict(dtrain)\n",
    "# #得到预测结果\n",
    "# train_pre = [round(res_train) for res_train in pre_train]\n",
    "# #获取样本标签\n",
    "# y_train = dtrain.get_label()\n",
    "# acc_train = accuracy_score(y_train,train_pre)\n",
    "# print(acc_train)\n",
    "\n",
    "# #训练结束之后进行预测（在测试样本上)\n",
    "# pre_test = bst.predict(dtest)\n",
    "# test_pre = [round(res_test) for res_test in pre_test]\n",
    "# y_test = dtest.get_label()\n",
    "# acc_test = accuracy_score(y_test,test_pre)\n",
    "# print(acc_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 模型可视化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAACTCAYAAAB1YlneAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XlcFFe2B/Bfs8kiIIpIWIIom5qERaMiGhdQB6PGDGrcGKMG0ffyMjp5ftCoCTpJwO0FJDPjGJdRM9nImChxR02iRkEFl6BikB1ZFFpFhLD0eX+0XXazNzRUd3O+n09/gOp7b526VRyK6qp7JUQExhhj+sVA7AAYY4xpHid3xhjTQ5zcGWNMD3FyZ4wxPcTJnTHG9BAnd8YY00Oc3BljTA9xcmeMMT3EyZ0xxvQQJ3fGGNNDRmIH8BSPgcAYY+qTNPUGn7kzxpge0pYzd8Y6TVVVFQ4fPozTp0/j9u3bOHfuHCoqKgAA/fr1g4ODAwYOHAh7e3s4Ojqid+/esLa2houLCyQSCWxsbAAAlpaWMDKS/wpJpVIAgEwmw8OHDyGVSlFYWIi7d+9CKpUiMzMTRUVFuHHjBjIyMgAAJiYm8PDwgL+/P0aPHg0/Pz8MGDBAhB5h+kiiJaNCakUQTPfdvXsXW7duRXx8PDIzMzF48GAsXrwYc+fOhYWFhdjhqSU+Ph6JiYn44osv8PjxY8yYMQMREREYPHiw2KEx7dHkZRlO7kynffvtt/jXv/6FQ4cOYeLEiVi/fj2GDh0qdlgdJi0tDQcOHMDatWvRvXt3bNmyBW+99ZbYYTHxcHJnuu/atWtYsWIF8vLysGrVKoSGhoodkta4f/8+QkJCcObMGZw4cQKBgYFih8Q6Byd3prtOnDiBCRMm4OLFixgyZIjY4Wi93NxceHp6IjAwED/88IPY4bCOxcmd6ZadO3ciLCwMNTU1MDQ0FDscnbZ48WJYWVlh8+bNYofCNI9vhWS6w8/PD4sWLYJMJlMrsW/duhWjRo2Cj48PvL29AcjvYlm2bFmLdZsqJ5FIIJHIf3/S0tLQq1cv3L17FwDw6NEjnD59WqWcoqximaKera2tUG/mzJkq6+jfv3+zsc2YMUO4rt7Yelqyfft2bN68GaGhoVi8eHGr6zEdR0Ta8GKMiIhMTEzaXNfDw4Nyc3OJiOjo0aNERCQ/xIkWLVqkUvaXX35R+bmpcsoqKiqIiMjCwoJkMhkZGBg0KLN+/XoiIiorKxParKiooOrqarKwsFBZFxFRcHBws9uUnZ0tfH///v0G61FXeno6PXz4sE11mVZqMq+KndQ5uTOBmZlZu+p7enoK3589e5aIniVSxVdHR0dauXJlg7r1yyk8ePBAZdmoUaPos88+IxcXF9q8eTM5OTlRUlISERHNmTOHiJ794VCuB4A+++yzRpefP3+ekpKS6K233iIiIjs7O+F9b29v4fvhw4e3ohda9oc//EEj7TCt0GRe5YeYmNZ4/fXXNdbWyZMnERAQgMOHD8PHxwdBQUHYvn07du/ejfHjxwvlvLy8AEClnDJra2uYmpoKP//888+QSCRwcXHBu+++i1mzZmHatGm4ePEirly5AkD+kNStW7dU2iEiSCSSBrctOjs7Y/jw4Xj55Zfx5MkTAEBxcbEQ1yuvvCKUtbW1BSC//XP69Olt7puFCxeirq6OP8vQd81l/k58MUbDhw+n6urqNtUtLS0lV1dXIiKKi4ujuLg4Wr16NRERzZw5U6WsVColW1vbBm0ol8vKyqLIyEgiIpo2bRoREe3YsUMoV1NTQ7Nnz6a4uDgqKysjIiKZTKbSHp6eoe/YsYMqKyuF9qF05j5o0CAikl/GcXV1pc8//5zGjx+v0s6jR49ULqV4eXm13CHNMDc3b1d9plWazKt8twzTKmfPnoWFhQV8fX3FDkXvSKVS2Nraoq6uTuxQmObw3TJMN4wcORIZGRmIjIwUOxS9kpiYiMjISE7sXQifuTOtVVVVBTMzM3z44YdYvXq12OHoHKlUit69e2PNmjX8x1J/8Zk70z2mpqYgIqxcuRJ2dnZYsWIFKisrxQ5L623cuBGmpqY4dOgQamtrObF3UXzmznROSkoKNm3ahPj4eOzatQuzZ8+GsbGx2GGJ4uDBg5g7dy4MDQ0RHx+vcicQ6xJ4+AGm/44cOYJ169YhKSkJgwcPxrJlyzBmzBg4OTmJHVq77N+/H8eOHcP27dvxwgsvYMqUKfjrX//KtzIygJM764pkMhnOnz+PPXv2YN++faiqqoKXlxf8/PwQHByMwMBAPPfcc2KHifv37+Onn35CSkoKEhIScP36dRgYGGDOnDkYPXo0QkND0a1bN7HDZNqJkztjrfH22w9RUvIEEyceRmFhIfLy8lBWVoYHDx4gNzcXtbW1wqxLlZWVqKqqAgBhdqbu3bvD2NgYNjY2cHJyQp8+fbDk3DlcfOcdODo6YsCAAejXr59o28f0Did3xlry00+AtTXg49MBjT//PJCb2wENsy6OkztjzXF1BbKyOnglgYHAyZMdvBLWxXByZ6wxKSmAVCrPu51i2jTg++87aWWsC+Dkzlh9c+bIr5ZER3fyii0sgIqKTl4p01Oc3BlTZmAAyGQiBvDf/w387W8iBsD0BCd3xhR69AAePBA7CgBGRkBtrdhRMN3Gww8wlpAA3LihJYkdkCf2DRvEjoLpKT5zZ12ClxdQb/4M7bFtG7BkidhRMN3El2VY12VqCjx91kh7GRoCPBwvUx9flmFdz6uvyq96aH1iB+SJ/dtvRf6Ul+kTTu5ML23fDuzeDUREaK7NtWvXNvnerFmz2r+C6dPlt0nyh6xMAzi5M70zdiyweDFw/PjnkEjk/7Xu2bMHlZWVGDJkCK5fv46LFy9ixYoVKC8vV6lbUVGB2bNnw9fXF2fPnhUmpa6rq8NPP/0EAAgLC0N5eblKG926dcOBAwfg5OSEXr164dKlSyAiLF26VIihVSorgQsXgN9/FxYRERYvXoz58+fj0qVLAICVK1diwoQJ7ekmpu+am2C1E1+MtdvevURZWarLVq1aJUxcPXDgQGF5fHw8+fv7N9oOnk5gDagemp6enirLlduoXwcAbd26tdF2WiUlhai8vNF1jx49WlgeEBCgfttMnzSZV/nMnekFHx/AzAzo21d1+ccff4z58+cDAG7cuCF8vXbtGv7nf/6nTWO9ExGcnJxabGPp0qVCebX5+gKensCjRw3eGjNmjPB9SEiI+m2zLoHvlmE6LS8PGDSo0RwoePLkCczNzZt8//bt2/Dw8NB4bLt27cLChQuRl5cHZ2fntjWSkwN06wbY22s2OKYv+G4Zpn9qa4HVq5tO7BcuXICbm1uziR0A7DsocS5atAgAcO/evbY34uIiH9WsoEBDUbGugs/cmU7asEH+3I+1tdiRdBKpFCgtBdzcxI6EaRd+iInpByL5qLkHDogdiQgqKoA7d4CXXhI7EqY9+LIM032XLsmH6O2SiR2Q3wP/v/8LJCWJHQnTAXzmznSCuzvw229iR6ElamqAn3/uxBlGmBbjM3emu956Sz6aI3vK2BjYtw84eFDsSJgW4zN3pnUUE2lUVcmHD4iNFTsiLUUE/PvfwMKF8tsl6z1ty7oE/kCV6QY/PyA1FTAxkd/i2K2b2BFpORMT+WUaQJ7sWVfDl2WY9rt1S57YAXm+ysoSNx6t95e/PEvsAPDRR+LFwrQOn7kzraE8vpahIfD3v8sHAGMtsLZ+9iTXo0eApaW48bDO1OSZu1FnRsH0z+PHj1FSUgKpVAqpVCosV/7exsZG+NqjRw9YW1sLoy0qbNkin1LUyAi4dw/o3r1z4tcLDx/KR5E0M5PfKyqVoqysDA8ePFDZL7W1tSqjYCr2i5GRESwtLWFjYwN7e/sWn+hluoHP3Lu43Nxc/Pjjjzhw4AAOHz6MqqoqODk5wcvLCyNGjIC/vz88PT3h6uraKfFUV1cjOTkZFy5cwK1bt3D+/HlhwK9x48bBz88Ps2fPhp+fX6fEI4aSkhJ8+eWXOHbsGK5cuYLCwkKYmpoiICAAw4YNg7u7O4YOHQpXV1eYmZl1WlylpaVITk5GRkYGkpKSkJycjN+e3p86YMAATJgwAdOnT8fw4cNhZMTnjZ2k6fGkmxsyshNfrIPIZDJau3Yt9e3blwwNDWnSpEl06tQpscPSmFu3btHGjRupd+/eBIDmz59PpaWlYofVrGPHjtGf/vQnAkDdu3en9957j4qKisQOq0PEx8fTyJEjCQC5u7vT+++/T7W1tWKHpU+azKtiJ3VO7h3gxIkT1KNHD+revTvFxMSIHU6nO3PmDIWGhhIAmjx5MlVWVooaT0FBAQUEBBAACg0NpcLCQlHjEds333xD/fr1IwAUExNDVVVVYoekyzi567Pg4GACQD/88IPYoWitnJwcmjlzJjk6OlJiYmKHrmvu3LkEgD799NMOXY8+KS4uJjc3NzIzM6P8/Hyxw9ElTeZVvuauo4gIpqamCA4Oxvfffy92ODrlzp07cHNzQ0ZGBvr376+RNmtqamBubo7x48fj8OHDGmmzq/rhhx+wYMECrFq1Cn/5y1/EDkfb8UNM+mLjxo2IiYnB3bt3xQ5FL6SlpWHevHlIVdxgr6bRo0dDIpHgxx9/1GxgDACQkpICf39/PHnyBIaGhmKHo434A1VdV1RURL17925TXQsLC5JKpeTv709RUVFUXFxMRES+vr4t1vXw8CAioj59+qgsV24rLS2NUlJShLlCi4uLqaSkhIiIvL29affu3bRkyRKhrpmZGRGRUG/cuHFEJL90opgrtLi4mNzd3ZuNzcXFhcrKyigmJqbR9aijqqqKVq1a1eryK1eupKVLl7ZpXTExMVRTU0NJSUmUk5MjLFfenqb6vb6mykVFRVFUVBQRNexnX19fYf9s2rSJdu/eTS+++KJQV7Efp02b1mD/KCjv46Yotoeo8eNAXZaWlm2uq8f4mruuc3JyanNdRcK0tLSktWvXEhHRwYMHKTg4WKVcXV0djRo1SmWZ4hc9KChImGi6rq6OLC0tKTMzU6WshYWFSh1l1tbWRES0fPlyIR6Fd955p0GsjbVRX1JSUoOyivW0xb59+1pd9rvvvmvzeszNzRss++ijj1S2R7nfFerq6sjAwEClXlPlgoKCGuwfRT/X3+9ERDdu3CAi+f6p3/fK+6f+epuivD3R0dHC8vbsHyKiwMDAdtXXQ5zcu7L6ydTX15e6detGRPKENnfuXHJwcGi0rkQiobNnz5K9vT3V1NSovLdt2zbh7D8tLU04I7e3tyci+QeLREQVFRWUlpZG9+/fbxBPWlqaSqJQvKdoAwBduHBBJeEpKO6CUV6elpbWfGe0ICwsrMUyY8aMadc6GkuMo0aNUtkeiURCDx48IHt7e/ruu++a3T+KcvVt27ZN+L5+PxOp7h8iEvZP/f5sLF7l/aP8amx7/Pz8hPW0d/8cOXKkXfX1UJN5lceW6YJkMhlGjRoFIsKMGTNw7949jB8/Hl9//XWjZQMCAlBUVNTgwZTw8HDIZDIAwMCBA1FUVATg2dOpirlD58+fj4EDB8LR0RESiQTp6elwezpd3MCBAxuNUfkJ16NHjz47G1Fyo5FxgJtqr7WsrKw0UkZdISEhKtsjk8lgbW2NoqIiTJs2rdn9oyhXX3h4uPC9cr8o+lF5/wAQ9g8AtfaPmZlZg/2jvD0zZ84U1tPe/VNSUtKu+l1Kc5m/E1+sBYaGhm2qd+3aNZJIJHTlyhUyNzenBQsWCO8tWLBAOHtTaOye8Jdffln4/vnnn6ddu3aRubk5HTx4kIiIPvjgA1qyZAmdOHGCiIjKyspo9erVQh3FpSAFxdm5op7ick9YWBgZGRkJbYSGhlJeXh4REdnY2DS4xrtjxw6aPn26UH/q1Kmt7ZZGvfHGG60uq9yP6gJAkZGRRCTf5g0bNhBRw+2ZM2dOg7rbt29vsEy5nGL/zJ49u8H+UbS7YMGCZvcPnp6B9+/fv8H+UcSq2Md5eXmUlZXV7P5paj1t8fHHH7e7DT3TZF7lu2V0iIGBgXCmzDQrOTkZRIRhw4a1qnxSUhJmzpyJnJycDo6MAfJhMvr3748a5VEwGcBD/uoHmUyGjIwMSCQS/P7772KHoxfef/99REVFYejQoa1O7AAwbNgw5OTkYNu2bejWrVuDS0ZMMyIiImBhYYHnn3+eE7ua+Mxdh02ZMgUnTpxASUlJh1wH1lcfffQR/va3v2n8WYEpU6bg6NGjyM7OhqOjo0bb7krmzZuHf//738jNzYWzs7PY4Wg7PnPXRwkJCaiqqsL3338POzs7hIaGih2S1tqwYQPMzMzw7rvvYuXKlR3yEFhCQgJqamqQmpqKQYMGYdSoUfwfVivFxcVBIpFgzpw5+Pzzz0FEnNjbq7kL8p34YhpUW1tL8+fPJwA0btw4Sk1NFTukTlNaWkpvvvkmmZiY0IgRI+g///mP2CEREVFsbCy5ubmRpaUlxcXFaf3IlR3l8uXLNGHCBAJAkyZN6rL9oEFN5lWxkzon906QlpZGK1euJBMTEwJA4eHhlJKSInZY7Zabm0uRkZHk5+dHAGj48OE6cx/0kSNH6JVXXiEA1K9fP4qNjdWLfaJQUVFBR48epZCQEAJAVlZWtHbtWuFhKaYxfLcMa1p1dTUSEhKQkJCAlJQUXL9+HcAaeHrGY/jw4fDw8MCLL74IBwcHeHl5wcLCQuMx5OXlIT09Hfn5+bh8+TJ++eUX3L59G48fP4aJiQmCg4MxdepU+Pn5wcfHR+Pr1zbFxcU4duwYUlNTcfz4ceGe8f79+8PNzQ1Dhw6Fi4sLvLy84Onp2WBmK02pra1Feno60tPTUVhYiOTkZDx/4QKiMzNRW1sLc3NzTJgwAb6+vpg4caJaH0ozjeCBw1jLqquBfv2AS5cAe3sNNbp6NU/crCW8vOSTkGvEypXyg4UnuRUbz6HKmhcfDyQmAvn5Gm44OVnDDbK2evNNDTYWHQ3cvAkYGAD87IVW4jP3Ls7PD3BzA775poNWIJEA2nGMsY4ikwEeHsAvvwB2dmJH09XwZRmmKjERGD++E/IuJ/euY8MG+deICHHj6Fr4Pncmd/UqMG4cEBTUSTl35MhOWAlryWefdcJKIiLkr+vXgT/8oRNWyJrDZ+5dxIYNgI2NCJ9//d//ATxVmugsLYHy8k5eaXU14OgIPB19knUI/kC1q6qpkSf1/HygRw8RAhg6VISVsvoqK0VYqYmJPLEbGAApKUAXuIVVm/BlGT2VlCS/3G1sDDx+LFJiB/iyjJaIihJx5TKZPLHb2ACxsSIG0rVwctdD4eFAbi5/jsmeeTofh7ikUmDaNCAwUOxIugS+5q5H7OyA7GzA3FzsSBhrwZYt8lcHDODWxfDdMvrMwAA4dw4oKdFsYtfYo+THj2umHdYm27Zptr3mjotWHzPvvitP7AkJQAcMZ8H4zF2nxcYCf/0rcP++2JG04MMPgTVrxI6iyxo+HLhwQewoWmBoCNTViR2FLuIzd32Sni5/AOnPf26Y2BUTHL/++uvCsj59+gCQz+Q0Y8aMBu1lZGTg0KFD6Nu3Lx4+fIgxY8aotNWzZ0+hnkwmQ0lJCXJzc0FE6N69O3JycoQzttLSUkTV//Tuu++Qn5+Pjz76SGgzMjISANCvX7+2dwRrlcuXVY+LsLAwAPLjwsPDQ9in9UkkEuG4SE1NbXBcKI4n5TYkEol8REKJRDgu4uLiUFpaKtRrVF0d8OQJYGbW7LYojqPPP/8cAODh4QGAj6NGNTdkZCe+WCtZWxPt3t30+6tWraL8/HyqqamhuLg4lfcAUHJycpN18XRi5Ppfu3fvLtRTLFNQTHatWL5//3767bffVBs2MGhQvry8vNH2mOaFhsqPC5lMRjU1NTRw4EDhvfj4+Cb3QVPHg/LX5ORklTYa+7p161bav39/6/f1wIFErZxMu/76uiAez13XGRoSnTvXurImJiZERFRZWUkLFy6k8ePHC+9ZWVnRpk2bVMo/efKEiJr+JS4rKyMrKyuh/Msvv0yPHj0iIiIXFxeVsjt27KBz9QM1NCQiIplMRg4ODlRdXU2Ojo5ERDRixIjWbRRrk0OHiO7dk38/evRoIiJydXUVjgszMzMiku/T+sdFS8mdSH48KbfRWNl169bRjh07KD09Xb3ghw0jeviwwWKZTEZERNXV1TRo0CAi6tLHESd3XTViBFFhoXp1YmJimn3/8ePH7YioaVFRUcL3Pj4+z96YOrVD1sda5u7e+rIddVwo/zFQOS5aKyCAqKREgxHplSbzKj+hqqUePwYGD5ZfX1eHh4cHbt++3WyZjphsAwDCwsJgamqKKVOmIDU19dkb/JSqaH77Tf5VzOPi7bffhqmpKeLi4lSPi9Y6e1Y+hMXPP8snG2CtwnfLaJmCAuBPfwJOnhQ7Eg26eVM+DsJLL4kdSZezbBkQEyN2FBoUGQm8+irw8stiR6It+G4ZXZCQIB+9T68SOwAMGMCTdohEoxN0aIPISPnJwnPPiR2J1uMzdy2wfbv8q17PWPbqq8ChQ2JHwfTJ3r3Ae+91wPRhOoUn69BGkZHyy9GTJokdSScwMJCfcXl6ih2J3qupkX9Wk5UFTJkidjSdIDZWPkHsxIliRyIGTu7aZuRIYP/+LjArWVAQ8OOPz54+1I7jTa8Ryf+WKri6ApmZ4sXTKU6eBF57TX4nQtfC19zF8tVXqj8PGiQ//s6e7QKJHZDP58ePlXcq5QdBJRL5xEh6LzBQ/ouVkAD07Pls+dWr4sUkMj5z72CGhoC3N3DqFDB2LNCWO8F0npGRPMEbGgK1tWJH0yUoEvyQIcDFi+LGIgrFWDUGBvLEf+KE2BF1FL4sIwZj42e5TDu6WSQPH8pnC5k+HYiPFzuaLkGR3Lv0caf8L4yJCfD77+LF0nH4skxnq3+S2tyYSXrP2lp+K1D9a1SsQ337rdgRiOjBA9Wfq6u73Fy+Xe7M/cqVK0hKSkJWVhZSUlKQlZWFvLw8/K7mX/U+ffrgueeeg6+vL3x9feHq6orAwECYmZlhyBDFSHzA6NHAf/0X0MhgjDrv8ePHuHr1KlJSUnD16lXk5OTg1q1buHfvntr9aWVlBUdHR/Tt2xfe3t7w8fGBu7s7/Pz8Oih63XHu3DlkZmbi6tWruHr1KrKyspCfn99CH38BYI7KEisrK9ja2sLT0xMuLi7w9fXFkCFD8MILL8DExKRDt0E0p08DISHy/x5lMoAIUqkUp0+fRlZWFlJTU5GamorCwkJIpVK1mra0tISzszNcXV2FHODv748BAwZ00MY0qunTxubGJujEl0Z8+umnNH78eAJA7u7uFBkZSXfu3NFU82q5e/cuff311zRlyhTq3r079ezZk6Kiouj27duixKOumzdv0qJFiwgAmZiY0IIFC2jnzp3CIGNiuHz5MsXGxtKIESMIALm5udHf//53KioqEi2m9rh58yZFR0eTnZ0dmZiYUFBQEO3cuZNu3rwpWkwlJSUUGxtLb7zxBgEgIyMjWr58uc70cVJSEq1atYpMTEzI3t6eZs2aRQkJCcJAd53t2rVrtGLFCurTpw8BoJCQENqzZ48mV6GfA4cdPHiQLCwsyMjIqMGIdtoqOTlZSE6LFy8WOxzBmTNnyNnZmQDQ8uXL6fLly2KH1GqJiYnk5+dHACg8PFwYNVDbPHnyhP74xz8SAAoMDNS5Pn7zzTcJAHl7e4v6B0jZ77//TtOmTSMANHnyZMrIyBA7pBbV1tbSe++9RwDI0dGRzp8/357m9CO5jxs3jiQSCSUmJrapF7SZ4j+O5sZb16SAgACSSCR05MiRTlmfGObOnUsAaO/evaKs/8MPPySJRELvvvuuKOvvDLt37yYANH369E5Zn2Ls+Pnz53fK+jrTvn37CADNmzdPnWq6m9zt7Ozou+++U2djdV5SUhL17NmTzpw5o7E2a2trhX+xu6Ly8nKaOnVqhyba2tpaeu2112j58uVa+99DR5s6dSrZ2tpqtM0dO3aQt7c33b17V6PtartPPvmEhg4d2lIx3Uvuc+fOJV9fXzW6Qv8kJyeTgdIsRm3l4OBAy5Yt00BEuq+uro4CAwPpp59+0mi7cXFx3MdKAgMDacyYMe1qo6Kignr27EnFxcUaiko39ejRgzZv3tzU27qT3CMiImjy5Mmt3vCvv/661WW1SUsTaiirqamhcePGqb2OvXv30s8//9zq8rrYl2rP7qOkW7du7f6Ae+/eveTm5tbq8rrywaQydY7V+tp6E0GfPn1aXVbsPr1z506n3Cjh5OREx48fr79YN5J7dXW12h8uoIW5E2NiYqimpoaSkpIavJeTkyPM6amwadMm2r17N7344otCGYW5c+eSqakpbd26VeW9X3/9VaWet7c37d69m5YsWSLUVV6PuzrT4zxVWlpKDg4OrS7/5z//mfbt26fWOlrqSyJqsi+ba6d+f7i4uFBZWRnFxMTQunXrhGUKimnbiOT/vSjaKysro7KyMiKSHytRUVHCWd3+/fvVvVZJREQhISFq11Gmbh9XVla2WMbCwoJef/31ZsvU72PFXUyfffaZsEz585uysjKhj5WP1XXr1qm8B4B+/fVXeu2114iIGvz3rLxvWiskJESt/5QkEola7be3T+vngH/84x9E9KyP/f39hVnGPDw8iOjZHx9fX18qUZolStHnje0P5X126NChNs37urbh3LK6kdxfeeUVtTdW0UHR0dEUGBhIRPKDSfGLrriD4ubNm8IrOztbqF9/x7ZEJpPRpEmTGn3vwIEDKj8r5ndUXk9ISAiNGjWqzZecFMmtJbW1tWq3rdyXgwcPJiLVvgwPD2/Ql43dNdHUQTto0CAKDg6mgoICoZyibF5eHhERxcbGNtgnACg4OFj4efbs2bR8+fIG61m5cqXa29xcvC3x8vJqUz0i+S//4MGD6dq1a5Senk7z5s2jq1evUnh4OFlbW1N2drZafezn5yd8f/fuXYqNjRV+VvRdXl4ezZ49u9F2FO/169ePoqKi6PHjxxQSEkLh4eHCsdrYvmmt1p607dq1q03tE7W9T+tv0+3btyk7O5siIiJIKpVSUFCQMC+cRbnXAAAHa0lEQVSxor9mzZpFRPLfCQsLiwax1N8fynWJ5JdKIyIi2rSdxsbGyj/qRnJvyz3Ujf1ilpeX0/r165t8X1ljB2t8fHyT5RVnmo2pP6t8Y+tRnCHl5+c3G1dTFi5c2GKZBhNUt1JLfdkcT09PYRub6/OwsDAqfDoprHJyLygooBkzZtDNmzepb9++lJGRQadOnaKbN28SAAoLCxPaUL5sV/8Xa+zYsS3GWl9bbqM9d+5cm/chkeqxQkS0fv16oS+aSqAt9bGFhYVwZ5ByElP0XUFBgdB3iuNTuf8nT55MvXr1ohEjRtDq1atVjlXlfdORt0G25T8Dhbb0aWPvXbhwgaqqqlT6WHGydPjwYfL29iY7OzuVOkOGDBGOV0X/KO8PItV99s0337T5pGLHjh3KP+pGct+4caPaG6roIMW/KzExMeTo6EhE8rNsxfuKRAKAAgIChPqKHVtTUyMsa+oAO3bsWLOxXL9+vck2FOtRnJm0NTGcPHmyTfVao6W+VC6n/GqqHWXl5eVEJL+8pPhwyNjYmKytrYlIdd83duZeWloq/Jybmyt8/+GHH6qUVT5jbS1nZ2e16xBRu+68UfRRWlpag4Tt6elJmzZtUruPiZ5dNlCm6LuNGzcKfac4PpX7Pzc3V2jXzs6Odu3aRSkpKSrHalvP3Kurq1tVbtGiRW1qn6jtfVp/mxT5QTmBK/dBeXk5jRo1ioiIUlJSiIhozZo1jcakvD+U1/ngwYMGfyBay9vbW/lH3UjuCQkJbdrYrqKxf/+aK9uWSzNdTWZmZqsTT32pqala08dZWVl05coVscNoVGZmpsoJVUv8/f07MJr2iYyMbFW5jtofDx8+JKlUqryoybyqdWPLdOvWDRkZGXB2dhYzHq1jZGSEyspKGBsbt7qOvb09ioqKOjAq3bZhwwacOnUKx44da3Mb9vb2+OKLLzBu3DgNRqY/amtr8eqrr6rVx5mZmRgxYgQfu/UcPXoUb7/9NjIyMpQX696QvwYGBrh+/ToGDRokRjxa47333oOzszOWLl3apvrFxcUYOnQofvvtN/0dHEpN+fn5GDlyJLKzszXWpqmpKR49esR9/FR+fj7c3Nzw5MkTGBi0bfDZ8PBwBAYGYubMmRqOTrd88cUXWLBgQVMDxenuwGFLliyhN954Q9QBqzrb8uXLycDAQOP/8o8dO5acnJyorq5Oo+3qgn379pGpqSldvHixQ9fz5ZdfkpOTk8YfktIFDx48IFNTU5UPvzXh/v37JJFI2nW/va7Jzc2l4cOH0yeffNJSUd245t6cPXv2kJmZGUVHR7emuM65f/8+LV68WOV+744SExNDFhYWetuXCjU1NRQdHU2mpqad/pTjnTt3SCKRUHR0tN4PRRAdHU3Gxsb0z3/+s0PXU1VVRVZWVhQREUFVVVUdui6xfPDBB2RgYEAHDx5sbRXdT+71RUZGkpWVFY0bN060gaHaKicnh9atW0cA6J133hHu8RbL2bNnycnJiby9vWnLli2ixtJW6enptGbNGgJAS5cupVu3bokdkoqzZ8/S9OnTycjIiLZs2aLy4IuuqKurozVr1pCjoyP5+PiI3scpKSk0Z84cMjIyoq1bt9LDhw9FjUddsbGx5O3tTX379qVPP/20rc3oX3JXVl1dLTzUYm5uTu+//36nja7YkvLycvr+++9p5MiRBID8/PzUGhJADJs3byZfX18CQKGhofTVV1+JHZKKrKws2r59Ow0cOJAA0NixY+nSpUtih6WWEydOCH08bNgw+uqrr7RqDJXq6mravn27MJzu2LFjaefOnWKH1awDBw6Qq6ur8NDbqVOnVG5xFtPJkydp2bJlZGhoSAYGBq16dqSVmsyrWvuBqiYVFBTg+PHjuHLlCo4fP45bt24BkM+k4urqCicnJzg5OaFXr16wtraGjY0NAAhfFTO0SKVS5OTk4P79+ygoKEBRUREyMzOFtsaOHQsfHx9MmDABAQEBHblJojp58iSuX7+OxMREXLt2DXl5eQCAHj16wN3dHba2tnB2doaLiwsMDQ1V+rOurg6PHj0CIO9PqVSK7OxslJaWoqCgANnZ2aisrIREIoGHhweCgoIwaNAgBAUFwd3dXbRt7my//vorEhMTcfXqVZw8eVKlj11cXODg4ABbW1s4ODjAxsam2WO2srIShYWFyMzMhFQqxY0bN1BZWQkA8PT0FPr3pZde0vvj9tixY7h+/TpOnz4tfEA5YMAA9OnTBw4ODujduzeee+65ZvtT8crMzER+fj6ysrKEtry9vTFx4kS89NJLmDRpklC/A+ne3TKMMcZaxBNkM8ZYV2IkdgBPNX2vJmOMMbXxmTtjjOkhTu6MMaaHOLkzxpge4uTOGGN6iJM7Y4zpIU7ujDGmhzi5M8aYHuLkzhhjeoiTO2OM6SFO7owxpoc4uTPGmB7i5M4YY3qIkztjjOkhTu6MMaaHOLkzxpge4uTOGGN6iJM7Y4zpIU7ujDGmhzi5M8aYHuLkzhhjeoiTO2OM6SFO7owxpoc4uTPGmB76fxcvIWHhAyzmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'\\n第一个参数为训练好的模型 第二个参数是要打印的树的索引，从0开始 第三个参数是打印的格式\\n'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#模型可视化\n",
    "from xgboost import plot_tree\n",
    "import matplotlib.pyplot as plt\n",
    "xgb.plot_tree(bst,num_trees=0,rankdir='TD')\n",
    "plt.show()\n",
    "'''\n",
    "第一个参数为训练好的模型 第二个参数是要打印的树的索引，从0开始 第三个参数是打印的格式\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## xgboost和sklearn结合 fit predict方式"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9777368340242591\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_svmlight_file\n",
    "num_round = 2\n",
    "path = './data/'\n",
    "X_train,y_train = load_svmlight_file(path+'agaricus.txt.train')#xgboost支持libsvm格式的文件\n",
    "X_test,y_test = load_svmlight_file(path+'agaricus.txt.test')\n",
    "\n",
    "def evaluate_sklearn(data,model,label):\n",
    "    pre = model.predict(data)\n",
    "    pre_res = [round(res) for res in pre]\n",
    "    acc = accuracy_score(label,pre_res)\n",
    "    return acc\n",
    "\n",
    "bst = XGBClassifier(max_depth=2,learning_rate=1,n_estimators=num_round,silent=True,objective='binary:logistic')\n",
    "'''\n",
    "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
    "       colsample_bytree=1, gamma=0, learning_rate=0.1, max_delta_step=0,\n",
    "       max_depth=3, min_child_weight=1, missing=None, n_estimators=100,\n",
    "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
    "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
    "       silent=True, subsample=1)\n",
    "'''\n",
    "bst.fit(X_train,y_train)\n",
    "acc_train2 = evaluate_sklearn(X_train,bst,y_train)\n",
    "print(acc_train2)\n",
    "#print(acc_test2)\n",
    "#0.9777368340242591\n",
    "#0.9929372025180409"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## train_test_split 将训练集划分为train_set和dev_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9925581395348837\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "path = './data/'\n",
    "num_round = 3\n",
    "X_train,y_train = load_svmlight_file(path+'agaricus.txt.train')\n",
    "X_test,y_test = load_svmlight_file(path+'agaricus.txt.test')\n",
    "\n",
    "X_train_apart,X_dev_apart,y_train_apart,y_dev_apart = train_test_split(X_train,y_train,test_size = 0.33,random_state = 2333)\n",
    "\n",
    "def evaluate_sklearn(data,model,label):\n",
    "    pre = model.predict(data)\n",
    "    pre_res = [round(res) for res in pre]\n",
    "    acc = accuracy_score(label,pre_res)\n",
    "    return acc\n",
    "\n",
    "bst = XGBClassifier(max_depth=2,learning_rate=1,n_estimators=num_round,silent=True,objective='binary:logistic')\n",
    "bst.fit(X_train_apart,y_train_apart)\n",
    "\n",
    "print(evaluate_sklearn(X_dev_apart,bst,y_dev_apart))\n",
    "\n",
    "#0.9925581395348837  num_round=3 "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 学习曲线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-error:0.046528\tvalidation_0-logloss:0.233576\tvalidation_1-error:0.046512\tvalidation_1-logloss:0.234042\n",
      "Multiple eval metrics have been passed: 'validation_1-logloss' will be used for early stopping.\n",
      "\n",
      "Will train until validation_1-logloss hasn't improved in 10 rounds.\n",
      "[1]\tvalidation_0-error:0.022003\tvalidation_0-logloss:0.137507\tvalidation_1-error:0.022791\tvalidation_1-logloss:0.13582\n",
      "[2]\tvalidation_0-error:0.006876\tvalidation_0-logloss:0.082731\tvalidation_1-error:0.007442\tvalidation_1-logloss:0.083814\n",
      "[3]\tvalidation_0-error:0.015127\tvalidation_0-logloss:0.056502\tvalidation_1-error:0.015349\tvalidation_1-logloss:0.057887\n",
      "[4]\tvalidation_0-error:0.006876\tvalidation_0-logloss:0.041372\tvalidation_1-error:0.007442\tvalidation_1-logloss:0.04389\n",
      "[5]\tvalidation_0-error:0.001146\tvalidation_0-logloss:0.027251\tvalidation_1-error:0.001395\tvalidation_1-logloss:0.029045\n",
      "[6]\tvalidation_0-error:0.001146\tvalidation_0-logloss:0.01632\tvalidation_1-error:0.001395\tvalidation_1-logloss:0.01748\n",
      "[7]\tvalidation_0-error:0.001146\tvalidation_0-logloss:0.012005\tvalidation_1-error:0.001395\tvalidation_1-logloss:0.013095\n",
      "[8]\tvalidation_0-error:0.001146\tvalidation_0-logloss:0.009363\tvalidation_1-error:0.001395\tvalidation_1-logloss:0.009657\n",
      "[9]\tvalidation_0-error:0.001146\tvalidation_0-logloss:0.007026\tvalidation_1-error:0.001395\tvalidation_1-logloss:0.007514\n",
      "[10]\tvalidation_0-error:0.001146\tvalidation_0-logloss:0.005933\tvalidation_1-error:0.001395\tvalidation_1-logloss:0.006733\n",
      "[11]\tvalidation_0-error:0.001146\tvalidation_0-logloss:0.004517\tvalidation_1-error:0.001395\tvalidation_1-logloss:0.00495\n",
      "[12]\tvalidation_0-error:0\tvalidation_0-logloss:0.003551\tvalidation_1-error:0\tvalidation_1-logloss:0.003908\n",
      "[13]\tvalidation_0-error:0\tvalidation_0-logloss:0.003133\tvalidation_1-error:0\tvalidation_1-logloss:0.003542\n",
      "[14]\tvalidation_0-error:0\tvalidation_0-logloss:0.002627\tvalidation_1-error:0\tvalidation_1-logloss:0.002896\n",
      "[15]\tvalidation_0-error:0\tvalidation_0-logloss:0.002192\tvalidation_1-error:0\tvalidation_1-logloss:0.002307\n",
      "[16]\tvalidation_0-error:0\tvalidation_0-logloss:0.001835\tvalidation_1-error:0\tvalidation_1-logloss:0.001969\n",
      "[17]\tvalidation_0-error:0\tvalidation_0-logloss:0.001552\tvalidation_1-error:0\tvalidation_1-logloss:0.001648\n",
      "[18]\tvalidation_0-error:0\tvalidation_0-logloss:0.001313\tvalidation_1-error:0\tvalidation_1-logloss:0.00141\n",
      "[19]\tvalidation_0-error:0\tvalidation_0-logloss:0.001208\tvalidation_1-error:0\tvalidation_1-logloss:0.001295\n",
      "[20]\tvalidation_0-error:0\tvalidation_0-logloss:0.001102\tvalidation_1-error:0\tvalidation_1-logloss:0.001181\n",
      "[21]\tvalidation_0-error:0\tvalidation_0-logloss:0.000978\tvalidation_1-error:0\tvalidation_1-logloss:0.001066\n",
      "[22]\tvalidation_0-error:0\tvalidation_0-logloss:0.000938\tvalidation_1-error:0\tvalidation_1-logloss:0.001029\n",
      "[23]\tvalidation_0-error:0\tvalidation_0-logloss:0.000897\tvalidation_1-error:0\tvalidation_1-logloss:0.000989\n",
      "[24]\tvalidation_0-error:0\tvalidation_0-logloss:0.000859\tvalidation_1-error:0\tvalidation_1-logloss:0.000939\n",
      "[25]\tvalidation_0-error:0\tvalidation_0-logloss:0.000849\tvalidation_1-error:0\tvalidation_1-logloss:0.000933\n",
      "[26]\tvalidation_0-error:0\tvalidation_0-logloss:0.000835\tvalidation_1-error:0\tvalidation_1-logloss:0.000924\n",
      "[27]\tvalidation_0-error:0\tvalidation_0-logloss:0.00081\tvalidation_1-error:0\tvalidation_1-logloss:0.000892\n",
      "[28]\tvalidation_0-error:0\tvalidation_0-logloss:0.000801\tvalidation_1-error:0\tvalidation_1-logloss:0.000881\n",
      "[29]\tvalidation_0-error:0\tvalidation_0-logloss:0.000794\tvalidation_1-error:0\tvalidation_1-logloss:0.000876\n",
      "[30]\tvalidation_0-error:0\tvalidation_0-logloss:0.000779\tvalidation_1-error:0\tvalidation_1-logloss:0.000857\n",
      "[31]\tvalidation_0-error:0\tvalidation_0-logloss:0.000768\tvalidation_1-error:0\tvalidation_1-logloss:0.000851\n",
      "[32]\tvalidation_0-error:0\tvalidation_0-logloss:0.00076\tvalidation_1-error:0\tvalidation_1-logloss:0.00084\n",
      "[33]\tvalidation_0-error:0\tvalidation_0-logloss:0.000755\tvalidation_1-error:0\tvalidation_1-logloss:0.000838\n",
      "[34]\tvalidation_0-error:0\tvalidation_0-logloss:0.000752\tvalidation_1-error:0\tvalidation_1-logloss:0.000831\n",
      "[35]\tvalidation_0-error:0\tvalidation_0-logloss:0.000747\tvalidation_1-error:0\tvalidation_1-logloss:0.000824\n",
      "[36]\tvalidation_0-error:0\tvalidation_0-logloss:0.000743\tvalidation_1-error:0\tvalidation_1-logloss:0.000821\n",
      "[37]\tvalidation_0-error:0\tvalidation_0-logloss:0.000739\tvalidation_1-error:0\tvalidation_1-logloss:0.000819\n",
      "[38]\tvalidation_0-error:0\tvalidation_0-logloss:0.000736\tvalidation_1-error:0\tvalidation_1-logloss:0.000819\n",
      "[39]\tvalidation_0-error:0\tvalidation_0-logloss:0.000731\tvalidation_1-error:0\tvalidation_1-logloss:0.000812\n",
      "[40]\tvalidation_0-error:0\tvalidation_0-logloss:0.000727\tvalidation_1-error:0\tvalidation_1-logloss:0.000805\n",
      "[41]\tvalidation_0-error:0\tvalidation_0-logloss:0.000725\tvalidation_1-error:0\tvalidation_1-logloss:0.000803\n",
      "[42]\tvalidation_0-error:0\tvalidation_0-logloss:0.000722\tvalidation_1-error:0\tvalidation_1-logloss:0.000805\n",
      "[43]\tvalidation_0-error:0\tvalidation_0-logloss:0.000719\tvalidation_1-error:0\tvalidation_1-logloss:0.000804\n",
      "[44]\tvalidation_0-error:0\tvalidation_0-logloss:0.000716\tvalidation_1-error:0\tvalidation_1-logloss:0.0008\n",
      "[45]\tvalidation_0-error:0\tvalidation_0-logloss:0.000712\tvalidation_1-error:0\tvalidation_1-logloss:0.000798\n",
      "[46]\tvalidation_0-error:0\tvalidation_0-logloss:0.000711\tvalidation_1-error:0\tvalidation_1-logloss:0.000795\n",
      "[47]\tvalidation_0-error:0\tvalidation_0-logloss:0.000708\tvalidation_1-error:0\tvalidation_1-logloss:0.000794\n",
      "[48]\tvalidation_0-error:0\tvalidation_0-logloss:0.000706\tvalidation_1-error:0\tvalidation_1-logloss:0.000786\n",
      "[49]\tvalidation_0-error:0\tvalidation_0-logloss:0.000704\tvalidation_1-error:0\tvalidation_1-logloss:0.000782\n",
      "[50]\tvalidation_0-error:0\tvalidation_0-logloss:0.000701\tvalidation_1-error:0\tvalidation_1-logloss:0.000781\n",
      "[51]\tvalidation_0-error:0\tvalidation_0-logloss:0.000699\tvalidation_1-error:0\tvalidation_1-logloss:0.000777\n",
      "[52]\tvalidation_0-error:0\tvalidation_0-logloss:0.000696\tvalidation_1-error:0\tvalidation_1-logloss:0.000776\n",
      "[53]\tvalidation_0-error:0\tvalidation_0-logloss:0.000694\tvalidation_1-error:0\tvalidation_1-logloss:0.000776\n",
      "[54]\tvalidation_0-error:0\tvalidation_0-logloss:0.000692\tvalidation_1-error:0\tvalidation_1-logloss:0.000772\n",
      "[55]\tvalidation_0-error:0\tvalidation_0-logloss:0.00069\tvalidation_1-error:0\tvalidation_1-logloss:0.000768\n",
      "[56]\tvalidation_0-error:0\tvalidation_0-logloss:0.000688\tvalidation_1-error:0\tvalidation_1-logloss:0.000767\n",
      "[57]\tvalidation_0-error:0\tvalidation_0-logloss:0.000686\tvalidation_1-error:0\tvalidation_1-logloss:0.000769\n",
      "[58]\tvalidation_0-error:0\tvalidation_0-logloss:0.000685\tvalidation_1-error:0\tvalidation_1-logloss:0.000769\n",
      "[59]\tvalidation_0-error:0\tvalidation_0-logloss:0.000683\tvalidation_1-error:0\tvalidation_1-logloss:0.000766\n",
      "[60]\tvalidation_0-error:0\tvalidation_0-logloss:0.000681\tvalidation_1-error:0\tvalidation_1-logloss:0.000765\n",
      "[61]\tvalidation_0-error:0\tvalidation_0-logloss:0.00068\tvalidation_1-error:0\tvalidation_1-logloss:0.000763\n",
      "[62]\tvalidation_0-error:0\tvalidation_0-logloss:0.000678\tvalidation_1-error:0\tvalidation_1-logloss:0.000763\n",
      "[63]\tvalidation_0-error:0\tvalidation_0-logloss:0.000677\tvalidation_1-error:0\tvalidation_1-logloss:0.000759\n",
      "[64]\tvalidation_0-error:0\tvalidation_0-logloss:0.000675\tvalidation_1-error:0\tvalidation_1-logloss:0.000758\n",
      "[65]\tvalidation_0-error:0\tvalidation_0-logloss:0.000674\tvalidation_1-error:0\tvalidation_1-logloss:0.000756\n",
      "[66]\tvalidation_0-error:0\tvalidation_0-logloss:0.000672\tvalidation_1-error:0\tvalidation_1-logloss:0.000752\n",
      "[67]\tvalidation_0-error:0\tvalidation_0-logloss:0.000671\tvalidation_1-error:0\tvalidation_1-logloss:0.00075\n",
      "[68]\tvalidation_0-error:0\tvalidation_0-logloss:0.000669\tvalidation_1-error:0\tvalidation_1-logloss:0.00075\n",
      "[69]\tvalidation_0-error:0\tvalidation_0-logloss:0.000668\tvalidation_1-error:0\tvalidation_1-logloss:0.000747\n",
      "[70]\tvalidation_0-error:0\tvalidation_0-logloss:0.000667\tvalidation_1-error:0\tvalidation_1-logloss:0.000745\n",
      "[71]\tvalidation_0-error:0\tvalidation_0-logloss:0.000666\tvalidation_1-error:0\tvalidation_1-logloss:0.000745\n",
      "[72]\tvalidation_0-error:0\tvalidation_0-logloss:0.000665\tvalidation_1-error:0\tvalidation_1-logloss:0.000742\n",
      "[73]\tvalidation_0-error:0\tvalidation_0-logloss:0.000664\tvalidation_1-error:0\tvalidation_1-logloss:0.00074\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[74]\tvalidation_0-error:0\tvalidation_0-logloss:0.000663\tvalidation_1-error:0\tvalidation_1-logloss:0.000741\n",
      "[75]\tvalidation_0-error:0\tvalidation_0-logloss:0.000662\tvalidation_1-error:0\tvalidation_1-logloss:0.000739\n",
      "[76]\tvalidation_0-error:0\tvalidation_0-logloss:0.000661\tvalidation_1-error:0\tvalidation_1-logloss:0.000737\n",
      "[77]\tvalidation_0-error:0\tvalidation_0-logloss:0.00066\tvalidation_1-error:0\tvalidation_1-logloss:0.000736\n",
      "[78]\tvalidation_0-error:0\tvalidation_0-logloss:0.000659\tvalidation_1-error:0\tvalidation_1-logloss:0.000734\n",
      "[79]\tvalidation_0-error:0\tvalidation_0-logloss:0.000658\tvalidation_1-error:0\tvalidation_1-logloss:0.000734\n",
      "[80]\tvalidation_0-error:0\tvalidation_0-logloss:0.000657\tvalidation_1-error:0\tvalidation_1-logloss:0.000733\n",
      "[81]\tvalidation_0-error:0\tvalidation_0-logloss:0.000657\tvalidation_1-error:0\tvalidation_1-logloss:0.000731\n",
      "[82]\tvalidation_0-error:0\tvalidation_0-logloss:0.000656\tvalidation_1-error:0\tvalidation_1-logloss:0.000731\n",
      "[83]\tvalidation_0-error:0\tvalidation_0-logloss:0.000656\tvalidation_1-error:0\tvalidation_1-logloss:0.000731\n",
      "[84]\tvalidation_0-error:0\tvalidation_0-logloss:0.000655\tvalidation_1-error:0\tvalidation_1-logloss:0.00073\n",
      "[85]\tvalidation_0-error:0\tvalidation_0-logloss:0.000654\tvalidation_1-error:0\tvalidation_1-logloss:0.000728\n",
      "[86]\tvalidation_0-error:0\tvalidation_0-logloss:0.000653\tvalidation_1-error:0\tvalidation_1-logloss:0.000727\n",
      "[87]\tvalidation_0-error:0\tvalidation_0-logloss:0.000653\tvalidation_1-error:0\tvalidation_1-logloss:0.000726\n",
      "[88]\tvalidation_0-error:0\tvalidation_0-logloss:0.000652\tvalidation_1-error:0\tvalidation_1-logloss:0.000726\n",
      "[89]\tvalidation_0-error:0\tvalidation_0-logloss:0.000652\tvalidation_1-error:0\tvalidation_1-logloss:0.000726\n",
      "[90]\tvalidation_0-error:0\tvalidation_0-logloss:0.000651\tvalidation_1-error:0\tvalidation_1-logloss:0.000725\n",
      "[91]\tvalidation_0-error:0\tvalidation_0-logloss:0.000651\tvalidation_1-error:0\tvalidation_1-logloss:0.000723\n",
      "[92]\tvalidation_0-error:0\tvalidation_0-logloss:0.00065\tvalidation_1-error:0\tvalidation_1-logloss:0.000723\n",
      "[93]\tvalidation_0-error:0\tvalidation_0-logloss:0.00065\tvalidation_1-error:0\tvalidation_1-logloss:0.000723\n",
      "[94]\tvalidation_0-error:0\tvalidation_0-logloss:0.000649\tvalidation_1-error:0\tvalidation_1-logloss:0.000721\n",
      "[95]\tvalidation_0-error:0\tvalidation_0-logloss:0.000649\tvalidation_1-error:0\tvalidation_1-logloss:0.000721\n",
      "[96]\tvalidation_0-error:0\tvalidation_0-logloss:0.000649\tvalidation_1-error:0\tvalidation_1-logloss:0.000721\n",
      "[97]\tvalidation_0-error:0\tvalidation_0-logloss:0.000648\tvalidation_1-error:0\tvalidation_1-logloss:0.000719\n",
      "[98]\tvalidation_0-error:0\tvalidation_0-logloss:0.000648\tvalidation_1-error:0\tvalidation_1-logloss:0.00072\n",
      "[99]\tvalidation_0-error:0\tvalidation_0-logloss:0.000647\tvalidation_1-error:0\tvalidation_1-logloss:0.000719\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHZVJREFUeJzt3X2QVPWd7/H3t7unuwdmmBlgFDMDgopcERLEkSSaZ4nB3KzkVqkxJnVdYy7lvUkle3OTLbZSFR+ye8vk3trdJFoxVILX3c3qGk3usimN0ZinukZhQKICoqNxYQR1AHkaZqafvvePPoNNMwM9zMMZzvm8qqboPv07p7+HA5/+ze+c/h1zd0REJB4SYRcgIiITR6EvIhIjCn0RkRhR6IuIxIhCX0QkRhT6IiIxotAXEYkRhb6ISIwo9EVEYiQVdgHVZs6c6XPnzg27DBGR08rGjRv3uHvrydpNutCfO3cunZ2dYZchInJaMbN/r6WdhndERGJEoS8iEiMKfRGRGJl0Y/oiIiORz+fp7u6mv78/7FImRDabpb29nbq6ulNaX6EvIqe17u5uGhsbmTt3LmYWdjnjyt3Zu3cv3d3dzJs375S2oeEdETmt9ff3M2PGjMgHPoCZMWPGjFH9VqPQF5HTXhwCf9Bo9zUyoV/sO8ihR26j95Wnwi5FRGTSikzov3XwMI1P/y1b1/8q7FJEJEb27t3LkiVLWLJkCbNmzaKtre3o81wuV9M2brzxRrZv3z7OlZZF5kRudmoDAKV8PM7gi8jkMGPGDDZv3gzArbfeSkNDA1/96lePaePuuDuJxND97HvuuWfc6xwUmZ5+Njul/CDfF24hIiJAV1cXixYt4uabb2bp0qXs3r2bVatW0dHRwYUXXsjtt99+tO373vc+Nm/eTKFQoLm5mdWrV/Oud72L9773vbz55ptjWldkevqpVIp+r1Poi8TYbf+2ha27Do7pNhe+Yxq3/NmFp7Tu1q1bueeee7j77rsBuOOOO5g+fTqFQoEPf/jDXH311SxcuPCYdQ4cOMAHP/hB7rjjDr7yla+wdu1aVq9ePer9GBSZnj7AgKWhoOEdEZkczj33XC655JKjz++77z6WLl3K0qVL2bZtG1u3bj1unfr6eq688koALr74Yl599dUxrSkyPX2AfjIkCurpi8TVqfbIx8vUqVOPPn7ppZf4zne+w/r162lubuazn/3skNfbp9Ppo4+TySSFQmFMa4pUTz9vGRJF9fRFZPI5ePAgjY2NTJs2jd27d/Poo4+GUkekevo5y5BUT19EJqGlS5eycOFCFi1axDnnnMNll10WSh3m7qG88XA6Ojr8VG+isv2vl5FLNbB49RNjXJWITFbbtm3jggsuCLuMCTXUPpvZRnfvONm60RreSWRJaXhHRGRYkQr9YjJDygfCLkNEZNKKVOgXElnSJYW+iMhwIhX6pVSWOvX0RUSGFa3QT2bJKPRFRIYVqdD3VD0Zr21WOxGROIpY6GfJoNAXkYkzFlMrA6xdu5bXX399HCsti9SXs6irp86K5HMD1KUzYVcjIjFQy9TKtVi7di1Lly5l1qxZY13iMSIV+lZXD0B/X69CX0RCd++993LXXXeRy+W49NJLufPOOymVStx4441s3rwZd2fVqlWceeaZbN68mU996lPU19ezfv36Y+bgGUvRCv10eU79gb7DNDZND7kaEZlwj6yG158b223OWgxX3jHi1Z5//nl+9rOf8eSTT5JKpVi1ahX3338/5557Lnv27OG558p17t+/n+bmZr73ve9x5513smTJkrGtv0qkQj+RLvf0B44cCbkSEYm7xx9/nA0bNtDRUZ4Zoa+vj9mzZ/Oxj32M7du38+Uvf5mPf/zjXHHFFRNaV8RCv9zTzw8cDrkSEQnFKfTIx4u787nPfY5vfvObx7327LPP8sgjj/Dd736Xhx56iDVr1kxYXZG6eicVhH6uTz19EQnX8uXLeeCBB9izZw9Qvspnx44d9PT04O5cc8013HbbbWzatAmAxsZGDh06NO51Raqnn8oM9vR7Q65EROJu8eLF3HLLLSxfvpxSqURdXR133303yWSSm266CXfHzPjWt74FwI033sjnP/95ncgdiWSmfJeawoB6+iIy8W699dZjnl9//fVcf/31x7V75plnjlt27bXXcu21145XaUdFangnnS339Ivq6YuIDKmm0DezFWa23cy6zOy427Kb2VfMbKuZPWtmvzKzsyteu8HMXgp+bhjL4qul68s9/WJOPX0RkaGcNPTNLAncBVwJLAQ+bWYLq5o9A3S4+zuBB4FvB+tOB24B3g0sA24xs5axK/9YdUFPv5TTLRNF4mSy3QFwPI12X2vp6S8Dutz9FXfPAfcDK6uK+LW7D3avnwLag8cfAx5z933u/hbwGLBiVBWfQLa+AYDSgEJfJC6y2Sx79+6NRfC7O3v37iWbzZ7yNmo5kdsG7Kx43k255z6cm4BHTrBu20gKHIlMMLzjeYW+SFy0t7fT3d1NT09P2KVMiGw2S3t7+8kbDqOW0Lchlg35kWpmnwU6gA+OZF0zWwWsApgzZ04NJQ1tMPQtrzF9kbioq6tj3rx5YZdx2qhleKcbmF3xvB3YVd3IzJYDXweucj96J5Oa1nX3Ne7e4e4dra2ttdZ+HEskGfA6vKCbo4uIDKWW0N8AzDezeWaWBq4D1lU2MLOLgB9QDvw3K156FLjCzFqCE7hXBMvGTb9lsIKGd0REhnLS4R13L5jZFymHdRJY6+5bzOx2oNPd1wH/C2gAfmJmADvc/Sp332dm36T8wQFwu7vvG5c9CeRIk1BPX0RkSDV9I9fdHwYerlr2jYrHy0+w7lpg7akWOFIDliFRVE9fRGQokfpGLkA+kSFZ1M3RRUSGEr3QtwypooZ3RESGErnQLySzJEsKfRGRoUQu9IuJDHUlDe+IiAwlcqFfSGZJu0JfRGQokQv9UjJLWj19EZEhRS/0U/WkUeiLiAwlcqHvqSwZz4VdhojIpBS90K+rJ0uOUin606yKiIxU5ELfUvXUWZGBAQ3xiIhUi1zoU1cPQH+f7pMrIlItcqGfSJdDf0ChLyJynAiGfvk+uQP9h0OuRERk8ols6Of71dMXEakWudBPZcrDOwp9EZHjRTD0y/fJzfdrTn0RkWoRDP1yT78woJ6+iEi1yIV+OtsAQHHgSMiViIhMPpEL/bps+URuMafQFxGpFrnQT9eXe/qu0BcROU7kQj8zpXwit5TXiVwRkWqRC/1sthz6nlPoi4hUi1zoD16ySUH3yRURqRa50CeRYIA6yGtMX0SkWvRCHxggjamnLyJynEiGfs7SJAoa0xcRqRbR0M+SKOomKiIi1SIa+mmSRQ3viIhUi2To5xNZUiWFvohItUiGfjGRoU49fRGR40Qy9AvJLKmSxvRFRKpFMvSLySxpV+iLiFSLZOiXkvXUKfRFRI4TzdBPZcl4LuwyREQmnZpC38xWmNl2M+sys9VDvP4BM9tkZgUzu7rqtaKZbQ5+1o1V4SfiqSxZ1NMXEamWOlkDM0sCdwEfBbqBDWa2zt23VjTbAfw58NUhNtHn7kvGoNbaperJkCNfLFGXjOQvMyIip6SWRFwGdLn7K+6eA+4HVlY2cPdX3f1ZoDQONY5cup60FekbUG9fRKRSLaHfBuyseN4dLKtV1sw6zewpM/vkUA3MbFXQprOnp2cEmx6a1ZVvjj7Qp5uji4hUqiX0bYhlPoL3mOPuHcD1wN+b2bnHbcx9jbt3uHtHa2vrCDY9tMRg6B9R6IuIVKol9LuB2RXP24Fdtb6Bu+8K/nwF+A1w0QjqOyWJdPnm6Orpi4gcq5bQ3wDMN7N5ZpYGrgNqugrHzFrMLBM8nglcBmw98Vqjl8yUe/r5/sPj/VYiIqeVk4a+uxeALwKPAtuAB9x9i5ndbmZXAZjZJWbWDVwD/MDMtgSrXwB0mtkfgV8Dd1Rd9TMukunyLRPzA7p7lohIpZNesgng7g8DD1ct+0bF4w2Uh32q13sSWDzKGkcslS0P7+T7NbwjIlIpkhexD94cvaievojIMSIZ+umgp1/MKfRFRCpFM/TrGwAoDug+uSIilaIZ+kFPv5RT6IuIVIpk6GeCnr7nNbwjIlIpoqGvMX0RkaFEMvStLhjeGdAlmyIilSIZ+iSSHGYqiYEDYVciIjKpRDP0gcOJRupyCn0RkUqRDf2+VCOZvEJfRKRSZEN/IDWNKcVDYZchIjKpRDb08+kmppYOhl2GiMikEtnQL2aaafTDlEojud+LiEi0RTb0PdtCE70cHsiHXYqIyKQR2dC3KS2krMTB/W+FXYqIyKQR2dBPTm0BoPfA6G+0LiISFZEN/VTDDAD6D+wJuRIRkckjsqGfbSyH/sChvSFXIiIyeUQ29OunzQSg0KvQFxEZFNnQb2guh37xiE7kiogMimzoT20qh74r9EVEjops6Ft6Cn2ksf79YZciIjJpRDb0AQ5bIynNtCkiclSkQ79X0yuLiBwj0qHfl5pGtqDQFxEZFOnQH6hr0vTKIiIVIh36hfQ0GkoKfRGRQZEO/VKmmWl+CHdNrywiAhEPfa9vIWt5ensPh12KiMikEOnQT0wpz7R5aL8mXRMRgYiHfmpqedK1I/s1vbKICEQ89NMN0wHoO6hJ10REIOKhnwlm2sxpemUREaDG0DezFWa23cy6zGz1EK9/wMw2mVnBzK6ueu0GM3sp+LlhrAqvxZRg0rWiplcWEQFqCH0zSwJ3AVcCC4FPm9nCqmY7gD8H/rlq3enALcC7gWXALWbWMvqya9PQ0gpoemURkUG19PSXAV3u/oq754D7gZWVDdz9VXd/FihVrfsx4DF33+fubwGPASvGoO6aNDQ0U/AE9Cn0RUSgttBvA3ZWPO8OltViNOuOWiKZ4KA1kND0yiIiQG2hb0Msq/UrrjWta2arzKzTzDp7esb28spDml5ZROSoWkK/G5hd8bwd2FXj9mta193XuHuHu3e0trbWuOnaHEk2ks4r9EVEoLbQ3wDMN7N5ZpYGrgPW1bj9R4ErzKwlOIF7RbBswvSnmqgvHJzItxQRmbROGvruXgC+SDmstwEPuPsWM7vdzK4CMLNLzKwbuAb4gZltCdbdB3yT8gfHBuD2YNmEydVNY0pRoS8iApCqpZG7Pww8XLXsGxWPN1Aeuhlq3bXA2lHUOCqFTDONhzThmogIRPwbuQCebaaBI3gxH3YpIiKhi3zoU1/+LljfoQkdVRIRmZQiH/qD0ysf1kybIiLRD/26hmB65QOaU19EJPKhn24sh36/Ql9EJPqhXx9Mr5w/rJk2RUQiH/pTg+mVC706kSsiEvnQb2guh75remURkeiHfmN9hv0+lUSfhndERCIf+omEscvOoL5358kbi4hEXORDH2Bfup2mPoW+iEgsQr+34WxmFt4ATcUgIjEXi9D3lnmkKJLftyPsUkREQhWL0M+ccR4Ae3e8EHIlIiLhikXoN7cvAODQ7hdDrkREJFyxCP229rn0eZp8T1fYpYiIhCoWod86LcsOZpHa/2rYpYiIhCoWoW9m7Em30XBEJ3JFJN5iEfoAh6fMZmZ+F5SKYZciIhKa2IR+sXkeaQqUDrwWdikiIqGJTeinzzgXgLe6t4dciYhIeGIT+tPeUb5s88BrumxTROIrNqF/1uxzGfAUA2++FHYpIiKhiU/ot0xlJ2eQ2P+nsEsREQlNbEI/lUzwZqqNqYd12aaIxFdsQh/g0JTZzMy9Bu5hlyIiEopYhX6+aS5ZBvBDr4ddiohIKGIV+qmZ5cs2NfGaiMRVrEK/8azzAV2rLyLxFavQP3POeRQ8Qf8bmm1TROIpVqHfPrOJnX4GqT3bwi5FRCQUsQr9bF2S51MXctb+TZp4TURiKVahD9DT+h6mlA7D68+GXYqIyISrKfTNbIWZbTezLjNbPcTrGTP7l+D1p81sbrB8rpn1mdnm4OfusS1/5KYs+BAAB7Y+Hm4hIiIhOGnom1kSuAu4ElgIfNrMFlY1uwl4y93PA/4O+FbFay+7+5Lg5+YxqvuULVqwgBdLbfS/+JuwSxERmXC19PSXAV3u/oq754D7gZVVbVYC9waPHwQuNzMbuzLHzgVnTaPTFtOypxMKubDLERGZULWEfhuws+J5d7BsyDbuXgAOADOC1+aZ2TNm9lsze/8o6x21ZMLYc8a7SZf64bXOsMsREZlQtYT+UD326slrhmuzG5jj7hcBXwH+2cymHfcGZqvMrNPMOnt6emooaXSmzP8QJTeObH9i3N9LRGQyqSX0u4HZFc/bgV3DtTGzFNAE7HP3AXffC+DuG4GXgfOr38Dd17h7h7t3tLa2jnwvRmjJ+XN53udqXF9EYqeW0N8AzDezeWaWBq4D1lW1WQfcEDy+GnjC3d3MWoMTwZjZOcB84JWxKf3ULW5v4ikW0bT3Gcj1hl2OiMiEOWnoB2P0XwQeBbYBD7j7FjO73cyuCpr9CJhhZl2Uh3EGL+v8APCsmf2R8gnem91931jvxEhlUkn2zHw3SS/AjqfCLkdEZMKkamnk7g8DD1ct+0bF437gmiHWewh4aJQ1joup899P7qkkdP2G9HmXh12OiMiEiN03cgctPa+NzX6exvVFJFbiG/pzWtjgF9CwbwsMHAq7HBGRCRHb0J+aSfFGSwcJirBzfdjliIhMiNiGPsC08y+l4Anyr/w+7FJERCZErEP/kvPn8Jyfw5GXfhd2KSIiEyLeoT+3hfV+AQ17/gj5vrDLEREZd7EO/SnpFPtmdJSv1+/eEHY5IiLjLtahD9B4/vspuTHQpSEeEYm+2If+xQvmstXPpvclncwVkeiLfehfNKeZTi6gsWcTFAbCLkdEZFzFPvSzdUn2tS6jznOw65mwyxERGVexD32ApgXle7scefG3IVciIjK+FPrAkgXn8UJpNr2ah0dEIk6hD7yzvYmn7J209GyA/oNhlyMiMm4U+kBdMsHus5aT8jze9XjY5YiIjBuFfuC8pR9hj0/jwKafhV2KiMi4UegHPrroHTxRupj6Vx/XpZsiElkK/UDzlDQ7Z11OpnQE/5O+qCUi0aTQrzDn4hX0eoZ9G38adikiIuNCoV9h+eKz+a0vIfPyL6BUCrscEZExp9Cv0DI1zaszP0xDfi+uWTdFJIIU+lXOvPjPyHuSPZ0a4hGR6FHoV/nwkvP5g19IdtuDcLgn7HJERMaUQr/K9KlpHj9rFXX5g+Tu+wwUcmGXJCIyZhT6Q7j2qqv4eulm0q89TeHf/gLcwy5JRGRMKPSHsKitiY9e+wW+W/gkqT/+GP/DXWGXJCIyJhT6w1ixaBapy7/OI8VLsF9+HX/6B2GXJCIyagr9E/ivH5rP7xb9T35ZvBh75C/pe+LbYZckIjIqCv0TMDP+5ppL2HH591lXuoz63/0Nu3/yNSjmwy5NROSUKPRPIpEwPv+hBcz9L//Ev6ZWcNaWNfR8+2L6Xngs7NJEREZMoV+jd86ezuVf+zH/dM636e3rp/7+q+n5wUpKT6+B1zbp0k4ROS2YT7LLETs6OryzszPsMk5oY9duNj/w13xi4OecafsBKKSmwCU3kbrsS9BwRsgVikjcmNlGd+84aTuF/qnpzxf55ZbX2fTccxx6+Sk+UHyaTySfopSo4/CFn6Fl2fXQthQSybBLFZEYUOhPoHyxxP/r2sNvnnySxX9ay0r7PSkr0ZtsYv+sS6k743ymzmxnyox2rHUBtMzVh4GIjKkxDX0zWwF8B0gCP3T3O6pezwD/AFwM7AU+5e6vBq/9FXATUAS+5O6Pnui9TsfQr7T38ACPb3yBg1se5Yw3f88l/jyzeIuEvf33PGAZ9tSfQ1/zfEozF5A96wKaZp3DtNY2bMpMSOhUi4iMzJiFvpklgReBjwLdwAbg0+6+taLNfwPe6e43m9l1wH9y90+Z2ULgPmAZ8A7gceB8dy8O936ne+hXKhRLvPD6IV7be5B9Pbs40rODxN7tTDvwImf1v8y51n30nMDRdUhw0Jo4lGqhPz2dXLqFYmYaZJuxTANWlyVRV3/0z2RdlmSmnlQmS116Cql0lmQygSVSJJIpEokElkySSKRIJpNYMkUymcIsgZkB5UtTBx9jiYofA+ztPwcdXT7Y5ugLbz8/ZrmIjLdaQz9Vw7aWAV3u/kqw4fuBlcDWijYrgVuDxw8Cd1o5RVYC97v7APAnM+sKtveHWnfkdJZKJljU1sSitiZgNvDuo6+5O3sO53jujdc51L2F/n3dFA++AYffIN2/h/rcXhqOvMW03m4a/TDTOELKTu8bu5SCD47KboZT9aFxAo6dYN3h16nFcO0qlx/bPTp5++Hf61SM7EP02LpP/QO45r+/Gj7ka9vvseksjGafx3NbJ7N7ygIWf+0X4/oetYR+G7Cz4nk3lelV1cbdC2Z2AJgRLH+qat226jcws1XAKoA5c+bUWvtpzcxobczQ2ng2nHf2CdsWS86hvhy9fb0U+vvIDRyh0N9HId9PMddHIXeE4kA/hVwfpfwAeAkvFaFUxL149DGlEngRSgXe/g3Pg/+NHkws5+BBO3h7GVS0K4GDUapYXI5kB+y43x4rtzHcf/2KD7Rh06HihYrtlN93iP+YJ/wt9tj1h25y7EfMSdUyVHrcPtQSKCffbmWL49/j1LZZe9SNw9/NKN51pNs54cZGuK3RfjwUm8Y//2oJ/aH2o/pvYrg2tayLu68B1kB5eKeGmmIlmTCap2ZonpoJuxQROc3Vcsawm/LYxKB2YNdwbcwsBTQB+2pcV0REJkgtob8BmG9m88wsDVwHrKtqsw64IXh8NfCEl8cP1gHXmVnGzOYB84H1Y1O6iIiM1EmHd4Ix+i8Cj1K+ZHOtu28xs9uBTndfB/wI+MfgRO0+yh8MBO0eoHzStwB84URX7oiIyPjSl7NERCKg1ks29S0gEZEYUeiLiMSIQl9EJEYU+iIiMTLpTuSaWQ/w76PYxExgzxiVc7qI4z5DPPc7jvsM8dzvke7z2e7eerJGky70R8vMOms5gx0lcdxniOd+x3GfIZ77PV77rOEdEZEYUeiLiMRIFEN/TdgFhCCO+wzx3O847jPEc7/HZZ8jN6YvIiLDi2JPX0REhhGZ0DezFWa23cy6zGx12PWMFzObbWa/NrNtZrbFzL4cLJ9uZo+Z2UvBny1h1zrWzCxpZs+Y2c+D5/PM7Olgn/8lmAU2Usys2cweNLMXgmP+3qgfazP778G/7efN7D4zy0bxWJvZWjN708yer1g25LG1su8G+fasmS091feNROgH9/G9C7gSWAh8Org/bxQVgP/h7hcA7wG+EOzrauBX7j4f+FXwPGq+DGyreP4t4O+CfX4LuCmUqsbXd4BfuPt/AN5Fef8je6zNrA34EtDh7osoz+x7HdE81v8HWFG1bLhjeyXlqennU77L4PdP9U0jEfpU3MfX3XPA4H18I8fdd7v7puDxIcoh0EZ5f+8Nmt0LfDKcCseHmbUD/xH4YfDcgI9QviczRHOfpwEfoDx1Oe6ec/f9RPxYU57yvT64IdMUYDcRPNbu/jvKU9FXGu7YrgT+wcueAprN7KxTed+ohP5Q9/E97l68UWNmc4GLgKeBM919N5Q/GIAzwqtsXPw98Je8fTPdGcB+dy8Ez6N4zM8BeoB7gmGtH5rZVCJ8rN39NeB/Azsoh/0BYCPRP9aDhju2Y5ZxUQn9mu7FGyVm1gA8BPyFux8Mu57xZGafAN50942Vi4doGrVjngKWAt9394uAXiI0lDOUYAx7JTAPeAcwlfLQRrWoHeuTGbN/71EJ/Vjdi9fM6igH/o/d/afB4jcGf90L/nwzrPrGwWXAVWb2KuWhu49Q7vk3B0MAEM1j3g10u/vTwfMHKX8IRPlYLwf+5O497p4HfgpcSvSP9aDhju2YZVxUQr+W+/hGQjCW/SNgm7v/bcVLlfcpvgH414mubby4+1+5e7u7z6V8bJ9w988Av6Z8T2aI2D4DuPvrwE4zWxAsupzyrUcje6wpD+u8x8ymBP/WB/c50se6wnDHdh3wn4OreN4DHBgcBhoxd4/ED/Bx4EXgZeDrYdczjvv5Psq/1j0LbA5+Pk55jPtXwEvBn9PDrnWc9v9DwM+Dx+cA64Eu4CdAJuz6xmF/lwCdwfH+v0BL1I81cBvwAvA88I9AJorHGriP8nmLPOWe/E3DHVvKwzt3Bfn2HOWrm07pffWNXBGRGInK8I6IiNRAoS8iEiMKfRGRGFHoi4jEiEJfRCRGFPoiIjGi0BcRiRGFvohIjPx/pO03EoXhxEcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAG9VJREFUeJzt3XuQXOV55/Hv0/eZ0WWk0SCBhCxxKccCyoo8YG5eh8tiRGLL2UBZ7LpCsFxa74YKMXFt5N2yuTh/wNbWYmyUsFojQlQpLoF4V5uFpWJD7cWOAQEKIBQFgRUYkJA0uo0uMz09/ewf58zQavV4WlKPzug9v0+Vak6ffk/3c3RUP73znstr7o6IiKRDJukCRETk1FHoi4ikiEJfRCRFFPoiIimi0BcRSRGFvohIiij0RURSRKEvIpIiCn0RkRTJJV1AvVmzZvmCBQuSLkNE5LTyyiuv7Hb37vHaTbrQX7BgARs2bEi6DBGR04qZ/VMz7TS8IyKSIgp9EZEUUeiLiKTIpBvTFxE5HkNDQ/T29jIwMJB0KadEqVRi3rx55PP5E9peoS8ip7Xe3l6mTp3KggULMLOky5lQ7k5fXx+9vb0sXLjwhD5DwzsiclobGBigq6sr+MAHMDO6urpO6rcahb6InPbSEPgjTnZfgwn9HR9s44U/v4v33nkr6VJERCatYEK/f+d7XLXtfvZsez3pUkQkRfr6+li8eDGLFy9mzpw5zJ07d/R1uVxu6jNuvfVWtmzZMsGVRoI5kZsrFAGolo8kXImIpElXVxcbN24E4K677mLKlCl861vfOqqNu+PuZDKN+9mPPPLIhNc5Ipiefq7YDkB1KB2XbYnI5LZ161YuvPBCvvGNb7BkyRK2b9/OypUr6enp4YILLuCee+4ZbXvllVeyceNGKpUKnZ2drFq1ik9/+tNcdtll7Ny5s6V1BdPTzxfbAHCFvkhq3f0/NvHWhwda+pmLzprGnV+84IS2feutt3jkkUd46KGHALj33nuZOXMmlUqFq666ihtvvJFFixYdtc3+/fv5/Oc/z7333ssdd9zB2rVrWbVq1Unvx4hgevojoV+tDCZciYhI5Nxzz+Xiiy8eff3YY4+xZMkSlixZwubNm3nrrWMvPGlra2Pp0qUAfOYzn2Hbtm0trSm4nj7q6Yuk1on2yCdKR0fH6PLbb7/NAw88wEsvvURnZydf/epXG15vXygURpez2SyVSqWlNQXT0y+W4uEd9fRFZBI6cOAAU6dOZdq0aWzfvp3nnnsukTqC6ekX8kWqblBRT19EJp8lS5awaNEiLrzwQs455xyuuOKKROowd0/ki8fS09PjJzqJypE7u/n7s27i0n/9py2uSkQmq82bN/OpT30q6TJOqUb7bGavuHvPeNsGM7wDULY8puEdEZExhRX6FLBhhb6IyFiCCv0hy5MZbu62ZxGRNAor9CmQqaqnLyIylrBCP1Mgq+EdEZExBRX6FSuQqWp4R0RkLGGFfqZATqEvIqdQKx6tDLB27Vp27NgxgZVGgrk5C2DYChSG+5MuQ0RSpJlHKzdj7dq1LFmyhDlz5rS6xKOEFfqZAnldpy8ik8Sjjz7K6tWrKZfLXH755Tz44INUq1VuvfVWNm7ciLuzcuVKZs+ezcaNG/nKV75CW1sbL7300lHP4GmloEK/mi2S86GkyxCRpDy7Cna80drPnHMRLL33uDd78803+fGPf8zPf/5zcrkcK1eu5PHHH+fcc89l9+7dvPFGVOe+ffvo7Ozkhz/8IQ8++CCLFy9ubf11ggr94UyBvGtMX0SS95Of/ISXX36Znp7oyQhHjhzh7LPP5gtf+AJbtmzh9ttv54YbbuC66647pXUFFfqeLZJXT18kvU6gRz5R3J2vfe1rfO973zvmvddff51nn32WH/zgBzz99NOsWbPmlNUV1NU71WyRAurpi0jyrr32Wp588kl2794NRFf5vPfee+zatQt356abbuLuu+/m1VdfBWDq1Kn090/8hShB9fTJFSmopy8ik8BFF13EnXfeybXXXku1WiWfz/PQQw+RzWZZsWIF7o6Zcd999wFw66238vWvf33CT+QG9Wjln/3XO7jig4fhzn1g1uLKRGQy0qOVIy19tLKZXW9mW8xsq5kdM0OvmRXN7In4/RfNbEHd+/PN7KCZHf/Fq8cjVwSgqikTRUQaGjf0zSwLrAaWAouAm81sUV2zFcBedz8PuB+4r+79+4FnT77cceRKAJQHFfoiIo0009O/BNjq7u+6exl4HFhW12YZ8Gi8/BRwjVk0vmJmXwbeBTa1puRfIR+H/sDhCf8qEZk8Jtsw9UQ62X1tJvTnAu/XvO6N1zVs4+4VYD/QZWYdwB8Dd59UlU3KxMM75UGFvkhalEol+vr6UhH87k5fXx+lUumEP6OZq3canRGt/9sdq83dwP3uftB+xYlVM1sJrASYP39+EyWN8TlxT39o8MgJf4aInF7mzZtHb28vu3btSrqUU6JUKjFv3rwT3r6Z0O8Fzq55PQ/4cIw2vWaWA6YDe4DPAjea2X8EOoGqmQ24+4O1G7v7GmANRFfvnMiOAGQU+iKpk8/nWbhwYdJlnDaaCf2XgfPNbCHwAbAc+Jd1bdYDtwB/B9wIPO/R71qfG2lgZncBB+sDv5VGQr9S1olcEZFGxg19d6+Y2W3Ac0AWWOvum8zsHmCDu68HHgbWmdlWoh7+8okseizZQhsAlbJ6+iIijTR1R667PwM8U7fuuzXLA8BN43zGXSdQ33HJ5ePr9NXTFxFpKKhn76inLyLyqwUV+rliFPq6I1dEpLGwQr8wEvqaPUtEpJGgQj8/2tPX8I6ISCNBhf7I8I5reEdEpKGgQr9QjK7Tdw3viIg0FFbol9qjhYp6+iIijQQV+sVCkYpnFPoiImMIKvTzWWOQPAxrnlwRkUaCCn0zo0wBq2hMX0SkkaBCH6BsOWxYoS8i0khwoT9EQaEvIjKG8ELfCmQU+iIiDYUZ+lWdyBURaSTI0M9W1dMXEWkkuNAfzuTJqqcvItJQcKFfsSI59fRFRBoKLvSHswVy6umLiDQUXuhnCuR8KOkyREQmpeBCv5otknf19EVEGgku9D1TIK+evohIQ8GFfjVbpIB6+iIijQQX+q7QFxEZU3ihnyuRZxiq1aRLERGZdMIL/WwxWtDzd0REjhFc6Fsumid3uKzZs0RE6oUX+vmop18ePJxwJSIik09woU8uCv2hAYW+iEi94EI/k4+Gd4YGjyRciYjI5BNc6NtI6GtMX0TkGMGFfqbQBsCQxvRFRI4RXOhn4zH9inr6IiLHCC/0i1FPf7isMX0RkXrBhX4ur+v0RUTG0lTom9n1ZrbFzLaa2aoG7xfN7In4/RfNbEG8/hIz2xj/+Xsz++3Wln+s3EhPf0h35IqI1Bs39M0sC6wGlgKLgJvNbFFdsxXAXnc/D7gfuC9e/ybQ4+6LgeuB/2JmuVYV30guPpFbHdLwjohIvWZ6+pcAW939XXcvA48Dy+raLAMejZefAq4xM3P3w+5eideXAG9F0b9KrhgN71Q1vCMicoxmQn8u8H7N6954XcM2ccjvB7oAzOyzZrYJeAP4Rs1/AqPMbKWZbTCzDbt27Tr+vahRiId3vKLQFxGp10zoW4N19T32Mdu4+4vufgFwMfBtMysd09B9jbv3uHtPd3d3EyWNbWRM34cU+iIi9ZoJ/V7g7JrX84APx2oTj9lPB/bUNnD3zcAh4MITLbYZxWJ79H0KfRGRYzQT+i8D55vZQjMrAMuB9XVt1gO3xMs3As+7u8fb5ADM7BPAJ4FtLal8DIVCnrJn9Tx9EZEGxr2Sxt0rZnYb8ByQBda6+yYzuwfY4O7rgYeBdWa2laiHvzze/EpglZkNAVXg37r77onYkRHFXIYBClBR6IuI1Gvq8kl3fwZ4pm7dd2uWB4CbGmy3Dlh3kjUel1zGKJPDFPoiIscI7o5cM6NMHtPwjojIMYILfYCyFcgo9EVEjhFk6A9RwIbLSZchIjLphBn6ViBTVU9fRKRekKFfyeTJanhHROQYYYa+FchWNbwjIlIvzNDPKPRFRBoJMvSHM0VyrtAXEakXaOgXFPoiIg0EGfrVTIG8hndERI4RZuhni+RR6IuI1As39H0o6TJERCadIEPfs0UKKPRFROqFGfq5IjmGYfiYmRlFRFItyNAnW4x+6q5cEZGjhBn6uTj09Ux9EZGjBBr60dzrlcHDCRciIjK5BBn6lo9Cf6h8JOFKREQmlzBDPx7eKQ8MJFyJiMjkEmToZ/JtAFTU0xcROUqgoR/19IcGFfoiIrWCDP1sYeRErkJfRKRWkKGv4R0RkcaCDP2Rnv7wkE7kiojUCjT0o57+cFmhLyJSK8jQzxXbAfDyoYQrERGZXIIM/XxpKqDQFxGpF2To59qmRAsKfRGRowQZ+sViG2XP4oMHky5FRGRSCTL024tZDlPChtTTFxGpFWTodxRyHKKk4R0RkTpBhn4pn+Gwq6cvIlIvyNA3M45YG5khPU9fRKRWU6FvZteb2RYz22pmqxq8XzSzJ+L3XzSzBfH6f25mr5jZG/HPq1tb/tgGM23kKgp9EZFa44a+mWWB1cBSYBFws5ktqmu2Atjr7ucB9wP3xet3A19094uAW4B1rSp8POVMG7lhhb6ISK1mevqXAFvd/V13LwOPA8vq2iwDHo2XnwKuMTNz99fc/cN4/SagZGbFVhQ+nqFsO4VhPXBNRKRWM6E/F3i/5nVvvK5hG3evAPuBrro2vwO85u6nZLbySq6NYlU9fRGRWrkm2liDdX48bczsAqIhn+safoHZSmAlwPz585soaXzDuQ6KVfX0RURqNdPT7wXOrnk9D/hwrDZmlgOmA3vi1/OAHwO/6+7vNPoCd1/j7j3u3tPd3X18ezCG4Vw7bQxAtdqSzxMRCUEzof8ycL6ZLTSzArAcWF/XZj3RiVqAG4Hn3d3NrBP4n8C33f1nrSq6GZ7viBZ02aaIyKhxQz8eo78NeA7YDDzp7pvM7B4z+1Lc7GGgy8y2AncAI5d13gacB3zHzDbGf85o+V40UtRD10RE6jUzpo+7PwM8U7fuuzXLA8BNDbb7E+BPTrLGE2KFqKfv5YMYs5MoQURk0gnyjlyATNzTHzzcn3AlIiKTR7Chny1FoT9waH/ClYiITB7Bhn4unj1LPX0RkY8FG/r59qinX1boi4iMCjb0C23TAKgcUeiLiIwIN/Q74tAfUOiLiIwINvTb4tCvap5cEZFRwYZ+e1sHFc8wPKDQFxEZEW7oF3Mc1jy5IiJHCTb0O4ojk6Orpy8iMiLY0C/mMhympHlyRURqBBv6ZsaAlcgMqacvIjIi2NAHGDRNji4iUivs0Nfk6CIiRwk69Iey7eRrJ0d3x1dfir+6LrmiREQSFHToR5Ojfxz6PrAP27WZt1/73wlWJSKSnKBDfzjbQdE/Dv2DfdsBGDrwUVIliYgkKuzQz7dT8gFwB+DA7g8AKA3sTrIsEZHEBB36FDrIUoXKAABH9kY9/Y7KniSrEhFJTNCh74WjJ0cf3BeF/vThvUmVJCKSqKBDf2Ry9Gr8eOVq/04A2hgEPX1TRFIo7NAvxlMmxhOp2MGdo++N9PpFRNIk6ND/eHL0AwDkjnx8AvdA34eJ1CQikqSgQz9XGpknNwr9tvJuPvJOAA4p9EUkhYIO/Xw8T+5gPDl6x9Ae/pEF0ToN74hICgUd+oX2qKdfGegHd6ZX99HXcS5VNyq6QUtEUijo0C+OTI5+pB+O7CVPhez0s+hjKtSc1BURSYugQ7/UPh2A4YFDDO7fAUCxcw576DzqpK6ISFoEHfpt7R1U3agO9o8+giE3bQ4HsjMoDuxKuDoRkVMv6NDvKBU4TBEGD3J4T3Titq3zTA7lZ9I+pEcxiEj6hB36xSyHKcHQodGrdTq6zmSwNCt6FEP8IDYRkbQIOvQL2WhydCsfYvjAR5Q9y4yuMxhu76ZIGQb7ky5RROSUCjr0zYwjViJbOQyHdtLHdLqmlqDjDACG+3UFj4ikS9ChD9Hk6NnKIXKHd9NHJ+2FHNlpswE42PdBwtWJiJxaTYW+mV1vZlvMbKuZrWrwftHMnojff9HMFsTru8zsBTM7aGYPtrb05gxm28lVjlAa3E1/dgYApRlnAnBQj2IQkZQZN/TNLAusBpYCi4CbzWxRXbMVwF53Pw+4H7gvXj8AfAf4VssqPk5DmTby1cN0DO3hUKELgPYZZwF6FIOIpE8zPf1LgK3u/q67l4HHgWV1bZYBj8bLTwHXmJm5+yF3/39E4Z+ISrad0vAhplX3US5FoT9j1myG3TRXroikTjOhPxd4v+Z1b7yuYRt3rwD7ga5mizCzlWa2wcw27NrV2pumhvPtdFb3kmOY4fboBG7X1Hb2MA36Ffoiki7NhL41WFd/gXszbcbk7mvcvcfde7q7u5vdrCnVfDxPLmBTotCf3pZnl3eS1aMYRCRlmgn9XuDsmtfzgPozoKNtzCwHTAcmxS2v1XzH6HJuahT6mYxxINtJYUChLyLp0kzovwycb2YLzawALAfW17VZD9wSL98IPO8+SW53rQn94syzRpcP5rvoKPclUZGISGJy4zVw94qZ3QY8B2SBte6+yczuATa4+3rgYWCdmW0l6uEvH9nezLYB04CCmX0ZuM7d32r9royhOGV0sWPGx6E/WOxian/8KAZrNDolIhKecUMfwN2fAZ6pW/fdmuUB4KYxtl1wEvWdtGwx6ukPeo6ZXbNG11fauin0D8HAfmjrTKo8EZFTKvg7cjOlqQDsZjpdU0qj6z0+qeuaTEVEUiT40B+ZHH23dzK9LT+6Pjs1ehTDgG7QEpEUCT/026Ke/oFsJ5nMx2P3xU49ikFE0if40C+0RfPkHsoffa9Y+8w5ABzZq56+iKRH8KFf7Ih6+oOlo0O/s2sOFc8wtF935YpIegQf+qVps3i/2s3uaRcctb5raolddEK/evoikh7Bh35bWzufKz/AjjOvOWr9zI4CH3oXhYMa0xeR9Ag+9KeW8pjB7Gmlo9YXc1l2ZrppP6LQF5H0aOrmrNPZ9LY8a3/vYpbMn3HMe/3FM5k2+DJUq5AJ/v8/EZHwe/oAV33yjKOu0R9RnjKXPENwSDdoiUg6pCL0x5LpjB4e6vveS7gSEZFTI9WhX+peAMChndsSrUNE5FRJdehPm3MOAP0f/TLhSkRETo1Uh/6ZZ3RzwNsZ3L0t6VJERE6JVIf+vBntfOCzsAO9SZciInJKpDr0p7fl+SjTTVE3aIlISqQ69AH6i3OYNqhHMYhIOqQ+9Msdc2n3Q9EMWiIigUt96DN6rf77CRciIjLxUh/6xa5PAHBAl22KSAqkPvRHrtU/sEOhLyLhS33on3HW2Qx6Ttfqi0gqpD70582cwnbv0pi+iKRC6kN/SjHHR5luCod0rb6IhC/1oQ/RtfpTda2+iKSAQh8YaD+LzuE9UCknXYqIyIRS6AN0zieD4wc+SLoSEZEJpdAHCl3zAdj34TsJVyIiMrEU+sC02dG1+vt3vJtwJSIiE0uhD8yauxCAI7v/KeFKREQmVi7pAiaDs2Z1stM7mf/u47D2lZP/wPYu+BdroNBx8p8lItJCCn2gvZDj4exvclnlddp3DpzUZ2V8mF8b/DuGfv4Q+d/4oxZVKCLSGgr9WPnS27l7y66T/hzH+ebh73D5//0++c+ugLbOFlQnItIa5u7jNzK7HngAyAI/cvd7694vAn8BfAboA77i7tvi974NrACGgT9w9+d+1Xf19PT4hg0bjn9PJpHvr3uaP3zna+xacjvdX7on6XJEJAXM7BV37xmv3bgncs0sC6wGlgKLgJvNbFFdsxXAXnc/D7gfuC/edhGwHLgAuB740/jzgvZ7v/NF/tYuY8praxjuP/nfHkREWqWZq3cuAba6+7vuXgYeB5bVtVkGPBovPwVcY2YWr3/c3Qfd/ZfA1vjzgtbZXiBz9b+nUB1g81Pq6YvI5NHMmP5coPYRlL3AZ8dq4+4VM9sPdMXrf1G37dwTrvY0cvWVn+Nnv7iWS7b9JdvueSHpckTkNLCj+0ou/TcPTeh3NBP61mBd/YmAsdo0sy1mthJYCTB//vwmSpr8zIzzb76PN//qj2F4MOlyROR0MO2sCf+KZkK/Fzi75vU8oP45xCNtes0sB0wH9jS5Le6+BlgD0YncZouf7GbPO5fZ33wq6TJEREY1M6b/MnC+mS00swLRidn1dW3WA7fEyzcCz3t0WdB6YLmZFc1sIXA+8FJrShcRkeM1bk8/HqO/DXiO6JLNte6+yczuATa4+3rgYWCdmW0l6uEvj7fdZGZPAm8BFeD33X14gvZFRETG0dR1+qdSCNfpi4icai27Tl9ERMKh0BcRSRGFvohIiij0RURSRKEvIpIik+7qHTPbBZzMFFazgN0tKud0kcZ9hnTut/Y5PY53vz/h7t3jNZp0oX+yzGxDM5cthSSN+wzp3G/tc3pM1H5reEdEJEUU+iIiKRJi6K9JuoAEpHGfIZ37rX1OjwnZ7+DG9EVEZGwh9vRFRGQMwYS+mV1vZlvMbKuZrUq6nolgZmeb2QtmttnMNpnZ7fH6mWb2t2b2dvxzRtK1TgQzy5rZa2b2N/HrhWb2YrzfT8SP/g6GmXWa2VNm9g/xMb8sDcfazL4Z//t+08weM7NSiMfazNaa2U4ze7NmXcPja5EfxPn2upktOdHvDSL0m5y8PQQV4I/c/VPApcDvx/u5Cvipu58P/DR+HaLbgc01r+8D7o/3ey+wIpGqJs4DwP9y918DPk2070EfazObC/wB0OPuFxI9zn05YR7rPweur1s31vFdSjQfyflEswz+2Yl+aRChT3OTt5/23H27u78aL/cThcBcjp6Y/lHgy8lUOHHMbB7wm8CP4tcGXA2MTE0W1H6b2TTgnxHNVYG7l919Hyk41kTzfLTFs/C1A9sJ8Fi7+/8hmn+k1ljHdxnwFx75BdBpZmeeyPeGEvqNJm8PegJ2M1sA/DrwIjDb3bdD9B8DcEZylU2Y7wP/DqjGr7uAfe5eiV+HdszPAXYBj8RDWj8ysw4CP9bu/gHwn4D3iMJ+P/AKYR/rWmMd35ZlXCih39QE7KEwsynA08AfuvuBpOuZaGb2W8BOd3+ldnWDpiEd8xywBPgzd/914BCBDeU0Eo9hLwMWAmcBHURDG/VCOtbNaNm/91BCv6kJ2ENgZnmiwP9Ld//rePVHI7/qxT93JlXfBLkC+JKZbSMauruaqOffGQ8BQHjHvBfodfcX49dPEf0nEPqxvhb4pbvvcvch4K+Bywn7WNca6/i2LONCCf1mJm8/7cXj2A8Dm939P9e8VTsx/S3Afz/VtU0kd/+2u89z9wVEx/Z5d/9XwAvAjXGzoPbb3XcA75vZJ+NV1xDNNR30sSYa1rnUzNrjf+8j+x3ssa4z1vFdD/xufBXPpcD+kWGg4+buQfwBbgD+EXgH+A9J1zNB+3gl0a90rwMb4z83EI1v/xR4O/45M+laJ/Dv4DeAv4mXzwFeArYCfwUUk66vxfu6GNgQH+//BsxIw7EG7gb+AXgTWAcUQzzWwGNE5y2GiHryK8Y6vkTDO6vjfHuD6OqmE/pe3ZErIpIioQzviIhIExT6IiIpotAXEUkRhb6ISIoo9EVEUkShLyKSIgp9EZEUUeiLiKTI/weF8mumZio8uAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "path = './data/'\n",
    "num_round = 100\n",
    "X_train,y_train = load_svmlight_file(path+'agaricus.txt.train')\n",
    "X_test,y_test = load_svmlight_file(path+'agaricus.txt.test')\n",
    "\n",
    "X_train_apart,X_dev_apart,y_train_apart,y_dev_apart = train_test_split(X_train,y_train,test_size = 0.33,random_state = 2333)\n",
    "\n",
    "def evaluate_sklearn(data,model,label):\n",
    "    pre = model.predict(data)\n",
    "    pre_res = [round(res) for res in pre]\n",
    "    acc = accuracy_score(label,pre_res)\n",
    "    return acc\n",
    "\n",
    "bst = XGBClassifier(max_depth=2,learning_rate=1,n_estimators=num_round,silent=True,objective='binary:logistic')\n",
    "eval_set = [(X_train_apart,y_train_apart),(X_dev_apart,y_dev_apart)]\n",
    "\n",
    "#early_stop\n",
    "bst.fit(X_train_apart,y_train_apart, early_stopping_rounds = 10,eval_metric = ['error','logloss'],eval_set = eval_set)\n",
    "\n",
    "#bst.fit(X_train_apart,y_train_apart, eval_metric = ['error','logloss'],eval_set = eval_set)\n",
    "\n",
    "results = bst.evals_result()\n",
    "epoch = len(results['validation_0']['error'])\n",
    "x_axis = range(0,epoch)#横轴表示树的个数\n",
    "fig,ax = plt.subplots()\n",
    "ax.plot(x_axis,results['validation_0']['logloss'],label='Train')\n",
    "ax.plot(x_axis,results['validation_1']['logloss'],label='Test')\n",
    "ax.legend()\n",
    "plt.ylabel = ('Log loss')\n",
    "plt.title= ('XGBoost Log loss')\n",
    "plt.show()\n",
    "\n",
    "\n",
    "fig,ax = plt.subplots()\n",
    "ax.plot(x_axis,results['validation_0']['error'],label='Train')\n",
    "ax.plot(x_axis,results['validation_1']['error'],label='Test')\n",
    "ax.legend()\n",
    "plt.ylabel = ('error')\n",
    "plt.title= ('XGBoost error')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K折交叉验证"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.69478528 1.         1.         1.         1.         1.\n",
      " 1.         1.         0.8156682  1.        ]\n",
      "CV ACC: 95.10% (10.16%)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "path = './data/'\n",
    "num_round = 100\n",
    "X_train,y_train = load_svmlight_file(path+'agaricus.txt.train')\n",
    "X_test,y_test = load_svmlight_file(path+'agaricus.txt.test')\n",
    "\n",
    "X_train_apart,X_dev_apart,y_train_apart,y_dev_apart = train_test_split(X_train,y_train,test_size = 0.33,random_state = 2333)\n",
    "\n",
    "def evaluate_sklearn(data,model,label):\n",
    "    pre = model.predict(data)\n",
    "    pre_res = [round(res) for res in pre]\n",
    "    acc = accuracy_score(label,pre_res)\n",
    "    return acc\n",
    "\n",
    "bst = XGBClassifier(max_depth=2,learning_rate=1,n_estimators=num_round,silent=True,objective='binary:logistic')\n",
    "\n",
    "#early_stop\n",
    "bst.fit(X_train_apart,y_train_apart)\n",
    "\n",
    "kfold = StratifiedKFold(n_splits=10,random_state=2333)\n",
    "results = cross_val_score(bst,X_train,y_train,cv = kfold)\n",
    "\n",
    "print(results)\n",
    "print('CV ACC: %.2f%% (%.2f%%)' % (results.mean()*100,results.std()*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GridSearch CV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise-deprecating',\n",
       "       estimator=XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n",
       "       colsample_bytree=1, gamma=0, learning_rate=1, max_delta_step=0,\n",
       "       max_depth=2, min_child_weight=1, missing=None, n_estimators=100,\n",
       "       n_jobs=1, nthread=None, objective='binary:logistic', random_state=0,\n",
       "       reg_alpha=0, reg_lambda=1, scale_pos_weight=1, seed=None,\n",
       "       silent=True, subsample=1),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'n_estimator': range(1, 50)}, pre_dispatch='2*n_jobs',\n",
       "       refit=True, return_train_score='warn', scoring='accuracy',\n",
       "       verbose=0)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sklearn\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "path = './data/'\n",
    "num_round = 100\n",
    "X_train,y_train = load_svmlight_file(path+'agaricus.txt.train')\n",
    "X_test,y_test = load_svmlight_file(path+'agaricus.txt.test')\n",
    "\n",
    "X_train_apart,X_dev_apart,y_train_apart,y_dev_apart = train_test_split(X_train,y_train,test_size = 0.33,random_state = 2333)\n",
    "\n",
    "def evaluate_sklearn(data,model,label):\n",
    "    pre = model.predict(data)\n",
    "    pre_res = [round(res) for res in pre]\n",
    "    acc = accuracy_score(label,pre_res)\n",
    "    return acc\n",
    "\n",
    "bst = XGBClassifier(max_depth=2,learning_rate=1,n_estimators=num_round,silent=True,objective='binary:logistic')\n",
    "#设置Boosting迭代计算次数\n",
    "param_test = {'n_estimator':range(1,50,1)}\n",
    "clf = GridSearchCV(estimator=bst,param_grid=param_test,scoring='accuracy',cv = 5)\n",
    "clf.fit(X_train,y_train)\n",
    "#clf.best_params_,clf.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "def evaluate_sklearn(data,model,label):\n",
    "    pre = model.predict(data)\n",
    "    pre_res = [round(res) for res in pre]\n",
    "    acc = accuracy_score(label,pre_res)\n",
    "    return acc\n",
    "\n",
    "print(evaluate_sklearn(X_test,clf,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
